37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* gpu_check.c - products on the GPU must equal the CPU's bit for bit, and
* where they do not, the report says which shader is at fault: random Q4_K,
* Q5_K, Q6_K, Q6_K_P (bit planes, at its three levels), Q8_0 and the blocks
* of 32 or 64 (Q4_0, IQ4_NL, Q4_1, Q5_1, MXFP4, Q5_0, Q2_0) matrices, the
* GPU's aligned copy of Q6_K (against the CPU's Q6_K), Q2_K, IQ2_XXS,
* IQ2_XS, IQ2_S, IQ3_XXS, IQ1_S and IQ1_M (Qwen3.8-Flash-Next's other
* files, unsloth's 1- and 2-bit ones), several shapes, vector counts and
* strides; for the blocks of 32 or 64 and Q8_0 also rows that end inside a
* group of 256 (qwen4exp's 640, its shared experts' down matrices Q8_0).
* Then bf16 weights on float vectors (janas_gpu_bf16_mv) against the CPU's
* eight lanes (bf16.h), with x and y where the GPU shares them and through
* its stages.
*
* Every product runs twice: all its rows on the GPU, with the name of the
* shader that computed them, and shared with the CPU as the engine does
* (the GPU's share adapting over four calls). Nothing stops at the first
* error: every wrong product is counted under its shader, with the worst
* difference in units in the last place (a few: the driver rounds a float
* operation differently, fused or reordered; many: wrong values). Then the
* tool isolates the culprits by itself: each wrong product again with the
* other shaders the engine could take for it, and with every other
* subgroup size the device allows. The report at the end is meant to be
* pasted whole into an issue. Skips (exit 0) when no GPU is usable.
*
* Usage: gpu_check
* GPU_CHECK_TYPES=Q6_K,Q8_0 checks those types only; GPU_CHECK_ENGINE=1
* the engine's pool (12 threads on CPUs 0-11, GPU_CHECK_PIN=1 the caller
* on CPU 0); GPU_CHECK_BIG_GB=n touches n GiB first, like the engine's
* expert cache; GPU_CHECK_GAP_US=n CPU-only work between the shared calls.
* JANAS_GPU_COPY=1 makes an integrated GPU work as a card with memory of
* its own.
*/
#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "common/aligned.h"
#include "llm/bf16.h"
#include "llm/gpu.h"
#include "llm/matvec.h"
#include "llm/quant.h"
#ifndef JANAS_VERSION
#define JANAS_VERSION "dev"
#endif
static uint64_t rng = 88172645463325252ull;
static uint32_t rnd(void)
{
rng ^= rng << 13;
rng ^= rng >> 7;
rng ^= rng << 17;
return (uint32_t)rng;
}
enum { MAXR = 4096, MAXC = 4096, MAXV = 64 };
/* the weights, made once and imported into every GPU tried */
static uint8_t *w, *w6, *wa, *wp, *w5, *w8, *w32[7], *wq[7], *wx, *w3[2];
static size_t wbytes, w6bytes, wabytes, wpbytes, w5bytes, w8bytes, w32bytes[7],
wqbytes[7], wxbytes, w3bytes;
static const uint8_t *pl1, *pl0;
static struct janas_block_q8k *x;
static float *ya, *yb;
static size_t ysz;
static struct janas_pool *pool;
/* blocks of 32 with four or five bits: Q4_0 and IQ4_NL (18 bytes: d, 16 of
nibbles), Q4_1 (20: d, m, ...), Q5_1 (24: d, m, qh, ...), MXFP4 (17: the
E8M0 byte, 16 of nibbles), Q5_0 (22: d, qh, ...), and Q2_0's blocks of 64
(18: d, two bits a weight; 9 bytes for 32 below) */
static const int t32[7] = {JANAS_Q4_0, JANAS_IQ4_NL, JANAS_Q4_1, JANAS_Q5_1,
JANAS_MXFP4, JANAS_Q5_0, JANAS_Q2_0};
static const size_t s32[7] = {18, 18, 20, 24, 17, 22, 9};
/* the 256-block types of iq2, iq3xxs, iq1 and q2k_matvec.comp */
static const int tq[7] = {JANAS_Q2_K, JANAS_IQ2_XXS, JANAS_IQ2_XS,
JANAS_IQ2_S, JANAS_IQ3_XXS, JANAS_IQ1_S,
JANAS_IQ1_M};
/* the last three: rows ending inside a group of 256, for the blocks of 32
or 64 and Q8_0 only (640: qwen4exp's experts' down matrices) */
static const size_t shapes[][2] = {{4096, 2048}, {1000, 256}, {128, 4096},
{2048, 512}, {4096, 4096}, {777, 1024},
{1500, 768}, /* odd blocks per row */
{2560, 640}, {300, 896}, {77, 128}};
static const size_t nvs[] = {1, 2, 3, 5, 8, 9, 16, 17, 33, 64};
enum {
N_SHAPES = sizeof(shapes) / sizeof(shapes[0]),
N_NVS = sizeof(nvs) / sizeof(nvs[0]),
N_KINDS = 25
};
/* 0 Q4_K, 1 Q6_K, 2-4 Q6_K_P with three, two and one planes, 5 Q5_K, 6
Q8_0, 7-10 Q4_0, IQ4_NL, Q4_1, Q5_1, 11 IQ4_XS, 12 Q3_K, 13 IQ3_S, 14 the
aligned Q6_K, 15 MXFP4, 16 Q5_0, 17 Q2_0, 18-24 Q2_K, IQ2_XXS, IQ2_XS,
IQ2_S, IQ3_XXS, IQ1_S, IQ1_M */
static const char *names[N_KINDS] = {
"Q4_K", "Q6_K", "Q6_K_P/3", "Q6_K_P/2", "Q6_K_P/1",
"Q5_K", "Q8_0", "Q4_0", "IQ4_NL", "Q4_1",
"Q5_1", "IQ4_XS", "Q3_K", "IQ3_S", "Q6_K_A",
"MXFP4", "Q5_0", "Q2_0", "Q2_K", "IQ2_XXS",
"IQ2_XS", "IQ2_S", "IQ3_XXS", "IQ1_S", "IQ1_M"};
static void fill(uint8_t *p, size_t n)
{
for (size_t i = 0; i < n; i++)
p[i] = (uint8_t)rnd();
}
static size_t page(size_t n)
{
return (n + 4095) / 4096 * 4096;
}
static void make_weights(void)
{
wbytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 144);
w = janas_aligned_alloc(4096, wbytes);
fill(w, wbytes);
struct janas_block_q4k *wb = (struct janas_block_q4k *)w;
for (size_t b = 0; b < wbytes / 144; b++) {
wb[b].d = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);
wb[b].dmin = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);
}
/* Q6_K weights too, with small scales */
w6bytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 210 + 4096);
w6 = janas_aligned_alloc(4096, w6bytes);
fill(w6, w6bytes);
struct janas_block_q6k *w6b = (struct janas_block_q6k *)w6;
for (size_t b = 0; b < (w6bytes - 4096) / 210; b++)
w6b[b].d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);
/* the same Q6_K in the GPU's aligned copy (gpu.h JANAS_Q6KA_BLOCK):
ql, qh, scales, d at byte 208 */
size_t nb6 = (w6bytes - 4096) / 210;
wabytes = page(nb6 * JANAS_Q6KA_BLOCK);
wa = janas_aligned_alloc(4096, wabytes);
memset(wa, 0, wabytes);
for (size_t b = 0; b < nb6; b++) {
uint8_t *d = wa + b * JANAS_Q6KA_BLOCK;
memcpy(d, w6b[b].ql, sizeof(w6b[b].ql));
memcpy(d + 128, w6b[b].qh, sizeof(w6b[b].qh));
memcpy(d + 192, w6b[b].scales, sizeof(w6b[b].scales));
memcpy(d + 208, &w6b[b].d, sizeof(w6b[b].d));
}
/* Q6_K_P: the high planes (82-byte blocks), then the two bare planes
(64 bytes a block), row after row as in an expert slot */
size_t pbase = page((size_t)MAXR * (MAXC / JANAS_QK) * 82 + 4096);
size_t pplane = (size_t)MAXR * (MAXC / JANAS_QK) * JANAS_Q6KP_PLANE;
wpbytes = pbase + 2 * pplane;
wp = janas_aligned_alloc(4096, wpbytes);
fill(wp, wpbytes);
struct janas_block_q6kp *wpb = (struct janas_block_q6kp *)wp;
for (size_t b = 0; b < (pbase - 4096) / 82; b++)
wpb[b].d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);
pl1 = wp + pbase;
pl0 = wp + pbase + pplane;
/* Q5_K */
w5bytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 176);
w5 = janas_aligned_alloc(4096, w5bytes);
fill(w5, w5bytes);
struct janas_block_q5k *w5b = (struct janas_block_q5k *)w5;
for (size_t b = 0; b < w5bytes / 176; b++) {
w5b[b].d = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);
w5b[b].dmin = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);
}
/* Q8_0: blocks of 32, eight to 256 weights */
w8bytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 272);
w8 = janas_aligned_alloc(4096, w8bytes);
fill(w8, w8bytes);
struct janas_block_q8_0 *w8b = (struct janas_block_q8_0 *)w8;
for (size_t b = 0; b < w8bytes / 34; b++)
w8b[b].d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);
/* Q3_K (d at byte 108) and IQ3_S (d at 0): 110-byte blocks */
w3bytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 110 + 4096);
for (int k = 0; k < 2; k++) {
w3[k] = janas_aligned_alloc(4096, w3bytes);
fill(w3[k], w3bytes);
for (size_t b = 0; b + 110 <= w3bytes; b += 110) {
uint16_t d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);
memcpy(w3[k] + b + (k == 0 ? 108 : 0), &d, 2);
}
}
/* IQ4_XS: 136-byte blocks, d, scales_h, scales_l, nibbles */
wxbytes = page((size_t)MAXR * (MAXC / JANAS_QK) * 136);
wx = janas_aligned_alloc(4096, wxbytes);
fill(wx, wxbytes);
for (size_t b = 0; b + 136 <= wxbytes; b += 136) {
uint16_t d = janas_fp32_to_fp16(0.00005f + (rnd() % 1000) * 1e-7f);
memcpy(wx + b, &d, 2);
}
/* the blocks of 32 or 64: random bytes with d (and m) set to small
scales, MXFP4's byte to powers of two from 2^-10 to 2^-3 (a subnormal
one, under 2^-125, a GPU may flush) */
for (int k = 0; k < 7; k++) {
w32bytes[k] = page((size_t)MAXR * (MAXC / JANAS_QK) * 8 * s32[k]);
w32[k] = janas_aligned_alloc(4096, w32bytes[k]);
fill(w32[k], w32bytes[k]);
size_t step = t32[k] == JANAS_Q2_0 ? 18 : s32[k];
for (size_t b = 0; b + step <= w32bytes[k]; b += step) {
if (t32[k] == JANAS_MXFP4) {
w32[k][b] = (uint8_t)(118 + rnd() % 8);
continue;
}
uint16_t d = janas_fp32_to_fp16(0.0005f + (rnd() % 1000) * 1e-6f);
memcpy(w32[k] + b, &d, 2);
if (t32[k] == JANAS_Q4_1 || t32[k] == JANAS_Q5_1) {
uint16_t m =
janas_fp32_to_fp16(-0.01f + (rnd() % 1000) * 2e-5f);
memcpy(w32[k] + b + 2, &m, 2);
}
}
}
/* the 256-block types: random bytes, the scales small - d at byte 0,
Q2_K's d and dmin at 80 and 82, IQ1_M's d in the top nibbles of its
four scale words (48-55) */
for (int k = 0; k < 7; k++) {
size_t bs = janas_qtype_block_size(tq[k]);
wqbytes[k] = page((size_t)MAXR * (MAXC / JANAS_QK) * bs + 4096);
wq[k] = janas_aligned_alloc(4096, wqbytes[k]);
fill(wq[k], wqbytes[k]);
for (size_t b = 0; b + bs <= wqbytes[k] - 4096; b += bs) {
uint16_t d = janas_fp32_to_fp16(0.0005f + (rnd() % 1000) * 1e-6f);
if (tq[k] == JANAS_Q2_K) {
uint16_t dm =
janas_fp32_to_fp16(0.0002f + (rnd() % 1000) * 1e-6f);
memcpy(wq[k] + b + 80, &d, 2);
memcpy(wq[k] + b + 82, &dm, 2);
} else if (tq[k] == JANAS_IQ1_M) {
for (int s = 0; s < 4; s++) {
uint16_t sc;
memcpy(&sc, wq[k] + b + 48 + 2 * s, 2);
sc = (uint16_t)((sc & 0x0fff) | ((d >> 4 * s & 15) << 12));
memcpy(wq[k] + b + 48 + 2 * s, &sc, 2);
}
} else {
memcpy(wq[k] + b, &d, 2);
}
}
}
size_t xblocks = (size_t)MAXV * 2 * (MAXC / JANAS_QK);
x = malloc(xblocks * sizeof(*x));
float *xf = malloc(MAXC * sizeof(float));
for (size_t v = 0; v < (size_t)MAXV * 2; v++) {
for (int i = 0; i < MAXC; i++)
xf[i] = (float)((int)(rnd() % 2001) - 1000) / 500.0f;
janas_q8k_quantize(xf, x + v * (MAXC / JANAS_QK), MAXC);
}
free(xf);
ysz = (size_t)MAXR * MAXV * 2;
ya = malloc(ysz * sizeof(float));
yb = malloc(ysz * sizeof(float));
}
static int import_all(struct janas_gpu *g)
{
for (int k = 0; k < 7; k++)
if (janas_gpu_import(g, wq[k], wqbytes[k]) != 0 ||
janas_gpu_import(g, w32[k], w32bytes[k]) != 0)
return -1;
return janas_gpu_import(g, w, wbytes) != 0 ||
janas_gpu_import(g, w6, w6bytes) != 0 ||
janas_gpu_import(g, wa, wabytes) != 0 ||
janas_gpu_import(g, wp, wpbytes) != 0 ||
janas_gpu_import(g, w5, w5bytes) != 0 ||
janas_gpu_import(g, w8, w8bytes) != 0 ||
janas_gpu_import(g, wx, wxbytes) != 0 ||
janas_gpu_import(g, w3[0], w3bytes) != 0 ||
janas_gpu_import(g, w3[1], w3bytes) != 0
? -1
: 0;
}
/* a product: weight kind, shape, vector count, strided or not */
struct kase {
int q6, s, k, strided;
};
/* The task of a case, for the CPU (gpu 0) or the GPU alone (1: the
aligned copy where the kind is Q6_K_A), and the stride of its results;
0 for a shape the kind does not take (a tail: blocks of 32 or 64, Q8_0
only). */
static int make_task(const struct kase *c, int gpu, struct janas_matvec_task *t,
size_t *ys)
{
int q6 = c->q6;
/* the blocks of 32 or 64: their table's index */
int b32 = q6 >= 7 && q6 <= 10 ? q6 - 7
: q6 == 15 ? 4
: q6 == 16 || q6 == 17 ? q6 - 11
: -1;
int iq = q6 >= 18 ? q6 - 18 : -1; /* the 256-block kinds */
size_t rows = shapes[c->s][0], cols = shapes[c->s][1];
if (cols % JANAS_QK && b32 < 0 && q6 != 6)
return 0;
/* the activations of a tail row: its group whole */
size_t nb = cols / JANAS_QK, nv = nvs[c->k],
nbx = (cols + JANAS_QK - 1) / JANAS_QK;
size_t skip = (size_t)c->s % 2; /* a row further in, every other shape */
size_t rb32 = b32 >= 0 ? janas_qtype_row_bytes(t32[b32], cols) : 0;
*t = (struct janas_matvec_task){
.type = iq >= 0 ? tq[iq]
: b32 >= 0 ? t32[b32]
: q6 == 14 ? (gpu ? JANAS_Q6_K_A : JANAS_Q6_K)
: q6 == 13 ? JANAS_IQ3_S
: q6 == 12 ? JANAS_Q3_K
: q6 == 11 ? JANAS_IQ4_XS
: q6 == 6 ? JANAS_Q8_0
: q6 == 5 ? JANAS_Q5_K
: q6 >= 2 ? JANAS_Q6_K_P
: q6 == 1 ? JANAS_Q6_K
: JANAS_Q4_K,
.w = iq >= 0 ? wq[iq] + skip * janas_qtype_block_size(tq[iq]) * nb
: b32 >= 0 ? w32[b32] + skip * rb32
: q6 == 14 ? (gpu ? wa + skip * JANAS_Q6KA_BLOCK * nb
: w6 + skip * 210 * nb)
: q6 >= 12 ? w3[q6 - 12] + skip * 110 * nb
: q6 == 11 ? wx + skip * 136 * nb
: q6 == 6 ? w8 + skip * janas_qtype_row_bytes(JANAS_Q8_0, cols)
: q6 == 5 ? w5 + skip * 176 * nb
: q6 >= 2 ? wp + skip * 82 * nb
: q6 == 1 ? w6 + skip * 210 * nb
: w + skip * 144 * nb,
.plane = {q6 == 2 || q6 == 3 ? pl1 + skip * JANAS_Q6KP_PLANE * nb
: NULL,
q6 == 2 ? pl0 + skip * JANAS_Q6KP_PLANE * nb : NULL},
.x = x,
.rows = rows,
.cols = cols,
.n_vec = nv,
.x_stride = c->strided ? 2 * nbx : 0,
.y_stride = c->strided ? rows + 5 : 0};
*ys = c->strided ? rows + 5 : rows;
return 1;
}
/* Units in the last place between two floats (NaN: the most). */
static uint64_t ulps(float a, float b)
{
if (isnan(a) || isnan(b))
return UINT64_MAX;
int32_t i, j;
memcpy(&i, &a, 4);
memcpy(&j, &b, 4);
int64_t oi = i >= 0 ? i : (int64_t)INT32_MIN - i,
oj = j >= 0 ? j : (int64_t)INT32_MIN - j;
return (uint64_t)(oi > oj ? oi - oj : oj - oi);
}
/* rows that differ from the CPU's, of all; the worst; the first that
differs (vector, row, the two values) */
struct diff {
size_t bad, all;
uint64_t ulp;
size_t v, r;
float a, b;
};
static void compare(size_t nv, size_t rows, size_t ys, struct diff *d)
{
memset(d, 0, sizeof(*d));
d->all = nv * rows;
for (size_t v = 0; v < nv; v++)
for (size_t r = 0; r < rows; r++) {
float a = ya[v * ys + r], b = yb[v * ys + r];
if (memcmp(&a, &b, 4) == 0)
continue;
if (!d->bad++) {
d->v = v;
d->r = r;
d->a = a;
d->b = b;
}
uint64_t u = ulps(a, b);
if (u > d->ulp)
d->ulp = u;
}
}
static void run_cpu(const struct kase *c)
{
struct janas_matvec_task t;
size_t ys;
make_task(c, 0, &t, &ys);
memset(ya, 0, ysz * sizeof(float));
t.y = ya;
janas_matvec_q4k_group(pool, &t, 1);
}
/* Every row on the GPU: 0 and the shader's name, -1 where the GPU does not
take the product (the CPU does it in the engine), -2 where the call
failed. */
static int run_gpu(struct janas_gpu *g, const struct kase *c, struct diff *d,
char *shader, size_t len)
{
struct janas_matvec_task t;
size_t ys;
make_task(c, 1, &t, &ys);
memset(yb, 0, ysz * sizeof(float));
t.y = yb;
size_t rows = t.rows;
int waited;
shader[0] = 0;
/* what the engine would not give the GPU (a misaligned block: only
some drivers read words off a 4-byte boundary) the CPU computes */
if (!janas_gpu_can(g, &t) || janas_gpu_begin(g, &t, &rows, 1) != 0)
return -1;
if (janas_gpu_finish(g, &waited) != 0)
return -2;
janas_gpu_last_shader(g, shader, len);
compare(t.n_vec, t.rows, ys, d);
return 0;
}
/* As the engine does: the rows shared between GPU and CPU, the share
adapting over several calls; results must not change. */
static void run_shared(struct janas_gpu *g, const struct kase *c,
struct diff *d)
{
struct janas_matvec_task t;
size_t ys;
make_task(c, 0, &t, &ys);
memset(yb, 0, ysz * sizeof(float));
t.y = yb;
const char *gap = getenv("GPU_CHECK_GAP_US");
for (int n = 0; n < 4; n++) {
janas_gpu_matvec_group(g, pool, &t, 1);
/* CPU-only work between calls, as in the engine */
if (gap) {
struct timespec a, b;
clock_gettime(CLOCK_MONOTONIC, &a);
do
clock_gettime(CLOCK_MONOTONIC, &b);
while ((b.tv_sec - a.tv_sec) * 1000000 +
(b.tv_nsec - a.tv_nsec) / 1000 <
atoi(gap));
}
}
compare(t.n_vec, t.rows, ys, d);
}
/* the products of a kind and a shader (or "shared with the CPU": right on
the GPU alone, wrong when shared), with up to MAX_EX wrong ones kept */
enum { MAX_GROUPS = 512, MAX_EX = 3 };
struct group {
int q6;
char shader[96];
int runs, wrong;
size_t rows_bad, rows_all;
uint64_t ulp;
int nex;
struct kase ex[MAX_EX];
struct diff dex[MAX_EX];
};
static struct group groups[MAX_GROUPS];
static int n_groups;
static struct group *group_of(int q6, const char *shader)
{
for (int i = 0; i < n_groups; i++)
if (groups[i].q6 == q6 && strcmp(groups[i].shader, shader) == 0)
return &groups[i];
if (n_groups == MAX_GROUPS)
return &groups[MAX_GROUPS - 1];
struct group *gr = &groups[n_groups++];
gr->q6 = q6;
snprintf(gr->shader, sizeof(gr->shader), "%s", shader);
return gr;
}
static void count(struct group *gr, const struct kase *c, const struct diff *d)
{
gr->runs++;
gr->rows_all += d->all;
if (!d->bad)
return;
gr->wrong++;
gr->rows_bad += d->bad;
if (d->ulp > gr->ulp)
gr->ulp = d->ulp;
if (gr->nex < MAX_EX) {
gr->ex[gr->nex] = *c;
gr->dex[gr->nex++] = *d;
}
}
static void print_case(const struct kase *c)
{
printf("%zux%zu nv %zu%s", shapes[c->s][0], shapes[c->s][1], nvs[c->k],
c->strided ? " strided" : "");
}
static void print_ulp(uint64_t u)
{
if (u == UINT64_MAX)
printf("NaN");
else
printf("%llu ulp", (unsigned long long)u);
}
/* a few units in the last place: rounding, not a wrong computation */
#define ROUNDING_ULP 64
static void print_diff(const struct diff *d)
{
uint32_t ua, ub;
memcpy(&ua, &d->a, 4);
memcpy(&ub, &d->b, 4);
printf("vector %zu row %zu: CPU %.9g (%08x) GPU %.9g (%08x), ", d->v, d->r,
d->a, ua, d->b, ub);
print_ulp(ulps(d->a, d->b));
}
static const char *verdict(uint64_t u)
{
return u <= ROUNDING_ULP
? "the last bits only: the driver rounds a float operation "
"differently (fused, reordered or flushed)"
: "wrong values: a shader or a driver bug";
}
/* Each wrong product again under one setting of the paths: what ran and
whether it was right. */
static void try_paths(struct janas_gpu *g, const struct group *gr,
const char *what, int row, int mvd)
{
int was_row = janas_gpu_set_path(g, JANAS_GPU_PATH_ROW, row),
was_mvd = janas_gpu_set_path(g, JANAS_GPU_PATH_MVD, mvd);
for (int e = 0; e < gr->nex; e++) {
struct diff d;
char sh[96];
run_cpu(&gr->ex[e]);
int rc = run_gpu(g, &gr->ex[e], &d, sh, sizeof(sh));
if (rc == 0 && strcmp(sh, gr->shader) == 0)
continue; /* the setting changes nothing here */
printf(" %s, ", what);
print_case(&gr->ex[e]);
if (rc != 0)
printf(": %s\n",
rc == -1 ? "the GPU does not take it" : "the call failed");
else if (!d.bad)
printf(": %s right\n", sh);
else {
printf(": %s wrong, %zu of %zu rows, worst ", sh, d.bad, d.all);
print_ulp(d.ulp);
printf("\n");
}
}
janas_gpu_set_path(g, JANAS_GPU_PATH_ROW, was_row);
janas_gpu_set_path(g, JANAS_GPU_PATH_MVD, was_mvd);
}
static int wanted(int q6)
{
const char *f = getenv("GPU_CHECK_TYPES");
if (!f || !*f)
return 1;
size_t n = strlen(names[q6]);
for (const char *p = f; (p = strstr(p, names[q6])) != NULL; p += n)
if ((p == f || p[-1] == ',') && (p[n] == 0 || p[n] == ','))
return 1;
return 0;
}
/* bf16 products: Qwen3.8-Flash-Next's mixes (10240 wide, 320 wide), a tail
of three past the last eight, a row shorter than eight; single vectors
too (as on a card with memory of its own). The count of wrong shapes;
*runs and *refused set. */
static int check_bf16(struct janas_gpu *g, int *runs, int *refused,
uint16_t **w16p, float **bxp, float **byp)
{
*runs = *refused = 0;
if (!janas_bf16_lanes())
return 0;
static const size_t bsh[][2] = {{320, 10240}, {10240, 320}, {2560, 2560},
{512, 2560}, {100, 1003}, {37, 13}};
static const size_t bnv[] = {1, 2, 3, 4, 5, 8, 17, 63};
enum { BW = 2560 * 2560, BX = 63 * 10240 };
size_t xbytes = page((size_t)BX * sizeof(float));
uint16_t *w16 = janas_aligned_alloc(4096, (size_t)BW * 2);
float *bx = janas_aligned_alloc(4096, xbytes),
*by = janas_aligned_alloc(4096, xbytes);
*w16p = w16;
*bxp = bx;
*byp = by;
float *bxs = malloc(xbytes), *bys = malloc(xbytes), *bref = malloc(xbytes);
for (size_t i = 0; i < BW; i++) {
float f = (float)((int)(rnd() % 20001) - 10000) * 1e-5f;
uint32_t u;
memcpy(&u, &f, 4);
w16[i] = (uint16_t)(u >> 16);
}
for (size_t i = 0; i < BX; i++)
bx[i] = bxs[i] = (float)((int)(rnd() % 2001) - 1000) / 500.0f;
janas_gpu_set_min_n(g, 1); /* single vectors too */
int bad = 0;
if (janas_gpu_import(g, w16, (size_t)BW * 2) != 0 ||
janas_gpu_import(g, bx, xbytes) != 0 ||
janas_gpu_import(g, by, xbytes) != 0) {
printf(" bf16: the import failed\n");
bad++;
}
for (size_t s = 0; !bad && s < sizeof(bsh) / sizeof(*bsh); s++)
for (size_t k = 0; k < sizeof(bnv) / sizeof(*bnv); k++)
for (int staged = 0; staged < 2; staged++) {
size_t rows = bsh[s][0], cols = bsh[s][1], nv = bnv[k];
float scale = staged ? 0.25f : 1.0f;
const float *xin = staged ? bxs : bx;
float *yout = staged ? bys : by;
for (size_t v = 0; v < nv; v++)
for (size_t r = 0; r < rows; r++)
bref[v * rows + r] =
janas_bf16_dot(w16 + r * cols, xin + v * cols,
(uint32_t)cols) *
scale;
memset(yout, 0, nv * rows * sizeof(float));
if (janas_gpu_bf16_mv(g, w16, xin, yout, rows, cols, nv,
scale) != 0) {
(*refused)++;
continue;
}
(*runs)++;
size_t nbad = 0, first = 0;
uint64_t worst = 0;
for (size_t i = 0; i < nv * rows; i++)
if (memcmp(&bref[i], &yout[i], 4) != 0) {
if (!nbad++)
first = i;
uint64_t u = ulps(bref[i], yout[i]);
worst = u > worst ? u : worst;
}
if (nbad) {
printf(" bf16 %zux%zu nv %zu %s: %zu of %zu wrong, "
"worst ",
rows, cols, nv,
staged ? "through the stages" : "in place", nbad,
nv * rows);
print_ulp(worst);
printf(", e.g. %zu: CPU %.9g GPU %.9g\n", first,
bref[first], yout[first]);
bad++;
}
}
free(bxs);
free(bys);
free(bref);
return bad;
}
int main(void)
{
/* the exact path: the fast prompt (on by default) rounds otherwise */
setenv("JANAS_GPU_FAST_PROMPT", "0", 1);
/* every shader as it is: the engine's check on opening would leave out
the ones this driver gets wrong */
setenv("JANAS_GPU_SELFCHECK", "0", 1);
const char *bg = getenv("GPU_CHECK_BIG_GB");
if (bg) {
size_t big = (size_t)atoi(bg) << 30;
uint8_t *b = janas_aligned_alloc(4096, big);
if (b)
memset(b, 1, big);
}
printf("gpu_check %s\n", JANAS_VERSION);
const char *off = getenv("JANAS_GPU");
if (off && strcmp(off, "0") == 0)
printf("(JANAS_GPU=0 turns the GPU off for the engine; gpu_check "
"opens it anyway, to check it)\n");
char err[256];
struct janas_gpu *g = janas_gpu_create(err, sizeof(err));
if (!g) {
printf("gpu_check: no GPU (%s), skipped\n", err);
return 0;
}
printf("device: %s\n", janas_gpu_about(g));
printf("weights in the %s\n", janas_gpu_shares_host(g)
? "host's memory, read in place"
: "card's own memory (a copy)");
make_weights();
if (import_all(g) != 0) {
printf("gpu_check: import failed\n");
return 1;
}
int cpus[12] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
int engine = getenv("GPU_CHECK_ENGINE") != NULL;
pool = engine ? janas_pool_create(12, cpus) : janas_pool_create(4, NULL);
if (engine && getenv("GPU_CHECK_PIN"))
janas_pin_current_thread(0);
/* every product, all its rows on the GPU, then shared with the CPU */
printf("\nproducts (each all on the GPU, then shared with the CPU):\n");
int runs = 0, wrong = 0, refused = 0, failed = 0;
for (int q6 = 0; q6 < N_KINDS; q6++) {
if (!wanted(q6))
continue;
int k_runs = 0, k_wrong = 0, k_refused = 0, k_shared = 0;
char used[512] = "";
for (int s = 0; s < N_SHAPES; s++)
for (int k = 0; k < N_NVS; k++)
for (int strided = 0; strided < 2; strided++) {
struct kase c = {q6, s, k, strided};
struct janas_matvec_task t;
size_t ys;
if (!make_task(&c, 0, &t, &ys))
continue;
run_cpu(&c);
struct diff d;
char sh[96];
int rc = run_gpu(g, &c, &d, sh, sizeof(sh));
k_runs++;
if (rc == -1) {
k_refused++;
} else if (rc == -2) {
failed++;
printf(" %s ", names[q6]);
print_case(&c);
printf(": the GPU's call failed\n");
} else {
struct group *gr = group_of(q6, sh);
count(gr, &c, &d);
k_wrong += d.bad != 0;
if (!strstr(used, sh) &&
strlen(used) + strlen(sh) + 3 < sizeof(used)) {
if (used[0])
strcat(used, ", ");
strcat(used, sh);
}
}
/* the aligned copy has no CPU counterpart to share
with; a product the GPU got wrong alone is not
counted twice */
if (q6 == 14 || (rc == 0 && d.bad))
continue;
run_shared(g, &c, &d);
if (d.bad) {
count(group_of(q6, "shared with the CPU"), &c, &d);
k_shared++;
}
}
runs += k_runs;
wrong += k_wrong + k_shared;
refused += k_refused;
printf(" %-9s %-6s %3d products", names[q6],
k_wrong || k_shared ? "WRONG" : "ok", k_runs);
if (k_wrong)
printf(", %d wrong on the GPU", k_wrong);
if (k_shared)
printf(", %d wrong shared with the CPU", k_shared);
if (k_refused)
printf(", %d left to the CPU", k_refused);
printf(" [%s]\n", used[0] ? used : "no GPU shader");
fflush(stdout);
}
/* the bf16 products */
int b16_runs, b16_refused;
uint16_t *w16 = NULL;
float *bx = NULL, *by = NULL;
int b16_bad = check_bf16(g, &b16_runs, &b16_refused, &w16, &bx, &by);
printf(" %-9s %-6s %3d products", "bf16",
b16_bad || b16_refused ? "WRONG" : "ok", b16_runs);
if (b16_bad)
printf(", %d wrong", b16_bad);
if (b16_refused)
printf(", %d refused", b16_refused);
printf("\n");
/* the wrong ones, by shader */
int culprits = 0;
for (int i = 0; i < n_groups; i++) {
struct group *gr = &groups[i];
if (!gr->wrong)
continue;
if (!culprits++)
printf("\nwrong products, by shader:\n");
printf(" %s on %s: %d of %d products wrong, %zu of %zu rows, "
"worst ",
gr->shader, names[gr->q6], gr->wrong, gr->runs, gr->rows_bad,
gr->rows_all);
print_ulp(gr->ulp);
printf(" - %s\n", verdict(gr->ulp));
for (int e = 0; e < gr->nex; e++) {
printf(" e.g. ");
print_case(&gr->ex[e]);
printf(": %zu of %zu rows, ", gr->dex[e].bad, gr->dex[e].all);
print_diff(&gr->dex[e]);
printf("\n");
}
}
/* isolating them: the other paths the engine could take */
if (culprits) {
printf("\nthe same products by other ways:\n");
for (int i = 0; i < n_groups; i++) {
struct group *gr = &groups[i];
if (!gr->wrong || strcmp(gr->shader, "shared with the CPU") == 0)
continue;
printf(" %s on %s:\n", gr->shader, names[gr->q6]);
try_paths(g, gr, "without the subgroup-per-row shaders", 0, 1);
try_paths(g, gr, "without the lanes-on-vectors shaders", 1, 0);
try_paths(g, gr, "without both", 0, 0);
}
}
unsigned sg = janas_gpu_subgroup(g), sg_min, sg_max;
janas_gpu_subgroup_range(g, &sg_min, &sg_max);
char name[256];
snprintf(name, sizeof(name), "%s", janas_gpu_name(g));
janas_gpu_destroy(g);
janas_aligned_free(w16);
janas_aligned_free(bx);
janas_aligned_free(by);
/* ... and with every other subgroup size the device allows */
if (culprits && sg_min < sg_max) {
printf("\nthe same products with other subgroup sizes:\n");
for (unsigned z = sg_min; z <= sg_max; z *= 2) {
if (z == sg)
continue;
char zs[16];
snprintf(zs, sizeof(zs), "%u", z);
setenv("JANAS_GPU_SG", zs, 1);
struct janas_gpu *g2 = janas_gpu_create(err, sizeof(err));
if (!g2 || import_all(g2) != 0) {
printf(" subgroups of %u: the GPU did not open (%s)\n", z,
g2 ? "import failed" : err);
if (g2)
janas_gpu_destroy(g2);
continue;
}
for (int i = 0; i < n_groups; i++) {
struct group *gr = &groups[i];
if (!gr->wrong ||
strcmp(gr->shader, "shared with the CPU") == 0)
continue;
for (int e = 0; e < gr->nex; e++) {
struct diff d;
char sh[96];
run_cpu(&gr->ex[e]);
int rc = run_gpu(g2, &gr->ex[e], &d, sh, sizeof(sh));
printf(" subgroups of %u, %s ", z, names[gr->q6]);
print_case(&gr->ex[e]);
if (rc != 0)
printf(": %s\n", rc == -1 ? "the GPU does not take it"
: "the call failed");
else if (!d.bad)
printf(": %s right\n", sh);
else {
printf(": %s wrong, %zu of %zu rows, worst ", sh, d.bad,
d.all);
print_ulp(d.ulp);
printf("\n");
}
}
}
janas_gpu_destroy(g2);
}
}
int bad = wrong || failed || b16_bad || b16_refused;
printf("\nsummary: %d products, %d wrong, %d left to the CPU, %d calls "
"failed; bf16 %d, %d wrong, %d refused; %d shader%s at fault\n",
runs, wrong, refused, failed, b16_runs, b16_bad, b16_refused,
culprits, culprits == 1 ? "" : "s");
if (bad)
printf("Please paste everything above into an issue at "
"https://github.com/prabanta-dev/janas/issues\n");
printf("gpu_check (%s, subgroups of %u): %d shapes, bf16 %d (%d "
"refused), %s\n",
name, sg, runs, b16_runs, b16_refused, bad ? "FAILED" : "ok");
janas_pool_destroy(pool);
janas_aligned_free(w);
janas_aligned_free(w6);
janas_aligned_free(wa);
janas_aligned_free(wp);
janas_aligned_free(w5);
janas_aligned_free(w8);
for (int k = 0; k < 7; k++) {
janas_aligned_free(w32[k]);
janas_aligned_free(wq[k]);
}
janas_aligned_free(wx);
janas_aligned_free(w3[0]);
janas_aligned_free(w3[1]);
free(x);
free(ya);
free(yb);
return bad ? 1 : 0;
}