Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

15.0 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * gpu_selfcheck.c - when the GPU opens, a product of every type it takes,
 * on the vector counts that pick each of its shaders, against the CPU's,
 * bit for bit (a fraction of a second, small random matrices). A shader
 * this device's driver gets wrong is left out: the subgroup-per-row ones,
 * the ones with the lanes on the vectors, q4k_mv (each family has another
 * shader for the same products); a type still wrong without them goes to
 * the CPU. On an Arc B580 a Q6_K shader gave text with no sense, issue
 * #43: with the check the engine answers right from the first run. The
 * result is kept in ~/.cache/janas/gpu_selfcheck under the device, its
 * driver's version, the subgroup size and Janas's version, so that the
 * check runs once per driver. JANAS_GPU_SELFCHECK=0 leaves it out
 * (gpu_check does, to see every shader), =2 runs it even when the cache
 * has the answer; JANAS_GPU_VERBOSE prints what it did. Not the fast
 * prompt's tiles (64 vectors and more, rounded otherwise: gpu_fast_check).
 */
#include "gpu.h"

#if defined(JANAS_GPU_SPV) && __has_include(<vulkan/vulkan.h>)
#include "gpu_impl.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#include "common/aligned.h"
#include "common/pool.h"
#include "matvec.h"
#include "quant.h"
#include "tuner.h" /* janas_cache_dir */

#ifndef JANAS_VERSION
#define JANAS_VERSION "dev"
#endif

enum { ROWS = 128, MAXC = 640, NV_MAX = 9, SLOT = 128 << 10 };

/* the types the products take, their block's bytes and where its scales
   are (offset of d, of a second scale or -1); 640 wide only for the types
   that sum a tail */
static const struct kind {
    int type;
    unsigned block, step; /* bytes a block of 256 weights, a block's */
    int d, m;
    int tail;
} kinds[] = {
    {JANAS_Q4_K, 144, 144, 0, 2, 0},    {JANAS_Q5_K, 176, 176, 0, 2, 0},
    {JANAS_Q6_K, 210, 210, 208, -1, 0}, {JANAS_Q6_K_P, 82, 82, 80, -1, 0},
    {JANAS_Q8_0, 272, 34, 0, -1, 1},    {JANAS_Q4_0, 144, 18, 0, -1, 1},
    {JANAS_IQ4_NL, 144, 18, 0, -1, 1},  {JANAS_Q4_1, 160, 20, 0, 2, 1},
    {JANAS_Q5_1, 192, 24, 0, 2, 1},     {JANAS_MXFP4, 136, 17, -1, -1, 1},
    {JANAS_Q5_0, 176, 22, 0, -1, 1},    {JANAS_Q2_0, 72, 18, 0, -1, 1},
    {JANAS_IQ4_XS, 136, 136, 0, -1, 0}, {JANAS_Q3_K, 110, 110, 108, -1, 0},
    {JANAS_IQ3_S, 110, 110, 0, -1, 0},  {JANAS_Q2_K, 84, 84, 80, 82, 0},
    {JANAS_IQ2_XXS, 66, 66, 0, -1, 0},  {JANAS_IQ2_XS, 74, 74, 0, -1, 0},
    {JANAS_IQ2_S, 82, 82, 0, -1, 0},    {JANAS_IQ3_XXS, 98, 98, 0, -1, 0},
    {JANAS_IQ1_S, 50, 50, 0, -1, 0},    {JANAS_IQ1_M, 56, 56, -2, -1, 0}};
enum { N_KINDS = sizeof(kinds) / sizeof(kinds[0]) };

/* vector counts: one (the row shaders, or the one-vector ones), 2 and 3
   (the row shaders' variants, the lanes on the vectors), 5 and 9 (the
   shaders of many vectors) */
static const size_t nvs[] = {1, 2, 3, 5, 9};
enum { N_NVS = sizeof(nvs) / sizeof(nvs[0]) };

static uint64_t rng = 0x9e3779b97f4a7c15ull;

static uint32_t rnd(void)
{
    rng ^= rng << 13;
    rng ^= rng >> 7;
    rng ^= rng << 17;
    return (uint32_t)rng;
}

/* random weights of kind k, scales small and normal */
static void fill(const struct kind *k, uint8_t *w, size_t bytes)
{
    for (size_t i = 0; i < bytes; i++)
        w[i] = (uint8_t)rnd();
    for (size_t b = 0; b + k->step <= bytes; b += k->step) {
        uint16_t d = janas_fp32_to_fp16(0.0005f + (rnd() % 1000) * 1e-6f);
        if (k->type == JANAS_MXFP4) {
            w[b] = (uint8_t)(118 + rnd() % 8);
        } else if (k->d == -2) {
            /* IQ1_M: d in the top nibbles of its four scale words */
            for (int s = 0; s < 4; s++) {
                uint16_t sc;
                memcpy(&sc, w + b + 48 + 2 * s, 2);
                sc = (uint16_t)((sc & 0x0fff) | ((d >> 4 * s & 15) << 12));
                memcpy(w + b + 48 + 2 * s, &sc, 2);
            }
        } else {
            memcpy(w + b + k->d, &d, 2);
        }
        if (k->m >= 0) {
            uint16_t m =
                k->type == JANAS_Q4_1 || k->type == JANAS_Q5_1
                    ? janas_fp32_to_fp16(-0.01f + (rnd() % 1000) * 2e-5f)
                    : janas_fp32_to_fp16(0.0002f + (rnd() % 1000) * 1e-6f);
            memcpy(w + b + k->m, &m, 2);
        }
    }
}

/* one product wrong: kind, width, vectors and the shader that ran it */
struct wrong {
    int k;
    size_t cols, nv;
    char shader[96];
};

/* Runs kind k's product of cols columns on nv vectors on the GPU, all
   its rows, and compares it with the CPU's: 1 right, 0 wrong (shader
   named), -1 not taken by the GPU. */
static int check_one(struct janas_gpu *g, struct janas_pool *pool,
                     const uint8_t *w, const struct janas_block_q8k *x,
                     float *ya, float *yb, int k, size_t cols, size_t nv,
                     char *shader, size_t len)
{
    size_t nbx = (cols + JANAS_QK - 1) / JANAS_QK;
    struct janas_matvec_task t = {.type = kinds[k].type,
                                  .w = w,
                                  .x = x,
                                  .rows = ROWS,
                                  .cols = cols,
                                  .n_vec = nv,
                                  .x_stride = nbx};
    t.y = ya;
    memset(ya, 0, ROWS * nv * sizeof(float));
    janas_matvec_q4k_group(pool, &t, 1);
    if (!janas_gpu_can(g, &t))
        return -1;
    t.y = yb;
    memset(yb, 0, ROWS * nv * sizeof(float));
    size_t rows = ROWS;
    int waited;
    if (janas_gpu_begin(g, &t, &rows, 1) != 0)
        return -1;
    if (janas_gpu_finish(g, &waited) != 0)
        return 0;
    janas_gpu_last_shader(g, shader, len);
    return memcmp(ya, yb, ROWS * nv * sizeof(float)) == 0;
}

/* The cache's key for this device and driver, and its file. */
static void cache_key(const struct janas_gpu *g, char *key, size_t len)
{
    snprintf(key, len, "%04x:%04x:%08x:sg%u:%s:%s%s", g->vendor_id,
             g->device_id, g->driver_version, g->sg,
             janas_gpu_shares_host(g) ? "shared" : "copy", JANAS_VERSION,
             g->xfma ? ":xfma" : "");
}

/* What a saved check says: 1 found (switches and types applied). */
static int load(struct janas_gpu *g, const char *key, int *flags)
{
    char path[512];
    if (janas_cache_dir(path, sizeof(path) - 32, 0) != 0)
        return 0;
    strcat(path, "/gpu_selfcheck");
    FILE *f = fopen(path, "r");
    if (!f)
        return 0;
    char line[512];
    int found = 0, row = 1, mvd = 1, mv = 1, off[16], n_off = 0;
    while (fgets(line, sizeof(line), f)) {
        char k[160], types[256] = "";
        int r, d, v;
        if (sscanf(line, "%159s row=%d mvd=%d mv=%d off=%255s", k, &r, &d, &v,
                   types) < 4 ||
            strcmp(k, key) != 0)
            continue;
        /* the last line of the key counts */
        found = 1;
        row = r;
        mvd = d;
        mv = v;
        n_off = 0;
        for (char *p = types; *p && n_off < 16;) {
            off[n_off++] = atoi(p);
            p = strchr(p, ',');
            if (!p)
                break;
            p++;
        }
        if (strcmp(types, "-") == 0)
            n_off = 0;
    }
    fclose(f);
    if (!found)
        return 0;
    g->row_on &= row;
    g->mvd_on &= mvd;
    g->mv_on &= mv;
    flags[0] = row;
    flags[1] = mvd;
    flags[2] = mv;
    for (int i = 0; i < n_off; i++)
        g->off_types[g->n_off++] = off[i];
    return 1;
}

static void save(const struct janas_gpu *g, const char *key, int row, int mvd,
                 int mv)
{
    char path[512];
    if (janas_cache_dir(path, sizeof(path) - 32, 1) != 0)
        return;
    strcat(path, "/gpu_selfcheck");
    FILE *f = fopen(path, "a");
    if (!f)
        return;
    fprintf(f, "%s row=%d mvd=%d mv=%d off=", key, row, mvd, mv);
    for (int i = 0; i < g->n_off; i++)
        fprintf(f, "%s%d", i ? "," : "", g->off_types[i]);
    fprintf(f, "%s\n", g->n_off ? "" : "-");
    fclose(f);
}

int janas_g_type_off(const struct janas_gpu *g, int type)
{
    for (int i = 0; i < g->n_off; i++)
        if (g->off_types[i] == type)
            return 1;
    return 0;
}

static const char *type_name(int type)
{
    switch (type) {
    case JANAS_Q4_K:
        return "Q4_K";
    case JANAS_Q5_K:
        return "Q5_K";
    case JANAS_Q6_K:
        return "Q6_K";
    case JANAS_Q6_K_P:
        return "Q6_K_P";
    case JANAS_Q8_0:
        return "Q8_0";
    case JANAS_Q4_0:
        return "Q4_0";
    case JANAS_IQ4_NL:
        return "IQ4_NL";
    case JANAS_Q4_1:
        return "Q4_1";
    case JANAS_Q5_1:
        return "Q5_1";
    case JANAS_MXFP4:
        return "MXFP4";
    case JANAS_Q5_0:
        return "Q5_0";
    case JANAS_Q2_0:
        return "Q2_0";
    case JANAS_IQ4_XS:
        return "IQ4_XS";
    case JANAS_Q3_K:
        return "Q3_K";
    case JANAS_IQ3_S:
        return "IQ3_S";
    case JANAS_Q2_K:
        return "Q2_K";
    case JANAS_IQ2_XXS:
        return "IQ2_XXS";
    case JANAS_IQ2_XS:
        return "IQ2_XS";
    case JANAS_IQ2_S:
        return "IQ2_S";
    case JANAS_IQ3_XXS:
        return "IQ3_XXS";
    case JANAS_IQ1_S:
        return "IQ1_S";
    case JANAS_IQ1_M:
        return "IQ1_M";
    }
    return "?";
}

/* The line for the user: what the check left out (row, mvd, mv: 0 where
   it turned that family off). */
static void make_note(struct janas_gpu *g, int row, int mvd, int mv)
{
    char *p = g->check_note;
    size_t len = sizeof(g->check_note);
    int n = 0;
    p[0] = 0;
    const char *fam[3] = {"the subgroup-per-row shaders",
                          "the lanes-on-vectors shaders", "q4k_mv"};
    int off[3] = {!row, !mvd, !mv};
    for (int i = 0; i < 3; i++)
        if (off[i] && (size_t)n < len)
            n +=
                snprintf(p + n, len - (size_t)n, "%s%s", n ? ", " : "", fam[i]);
    for (int i = 0; i < g->n_off && (size_t)n < len; i++)
        n += snprintf(p + n, len - (size_t)n, "%s%s",
                      i   ? " "
                      : n ? "; on the CPU: "
                          : "on the CPU: ",
                      type_name(g->off_types[i]));
    if (p[0] && (size_t)n < len)
        snprintf(p + n, len - (size_t)n, " (wrong on this driver)");
}

/* Every product once; the wrong ones in bad (up to max), their count
   returned. Only the cases in only (n_only of them), where given. */
static int run_all(struct janas_gpu *g, struct janas_pool *pool, uint8_t *w,
                   const struct janas_block_q8k *x, float *ya, float *yb,
                   const struct wrong *only, int n_only, struct wrong *bad,
                   int max)
{
    int n = 0;
    for (int k = 0; k < N_KINDS; k++)
        for (int c = 0; c < 1 + kinds[k].tail; c++)
            for (int v = 0; v < N_NVS; v++) {
                size_t cols = c ? MAXC : 512, nv = nvs[v];
                int want = !only;
                for (int i = 0; only && i < n_only; i++)
                    want |= only[i].k == k && only[i].cols == cols &&
                            only[i].nv == nv;
                if (!want)
                    continue;
                char sh[96];
                if (check_one(g, pool, w + (size_t)k * SLOT, x, ya, yb, k, cols,
                              nv, sh, sizeof(sh)) == 0) {
                    if (n < max)
                        bad[n] = (struct wrong){.k = k, .cols = cols, .nv = nv};
                    if (n < max)
                        snprintf(bad[n].shader, sizeof(bad[n].shader), "%s",
                                 sh);
                    n++;
                }
            }
    return n < max ? n : max;
}

void janas_g_selfcheck(struct janas_gpu *g)
{
    const char *e = getenv("JANAS_GPU_SELFCHECK");
    int mode = e ? atoi(e) : 1;
    if (mode == 0 || g->broken)
        return;
    char key[160];
    cache_key(g, key, sizeof(key));
    int flags[3];
    if (mode == 1 && load(g, key, flags)) {
        make_note(g, flags[0], flags[1], flags[2]);
        return;
    }
    struct timespec t0, t1;
    clock_gettime(CLOCK_MONOTONIC, &t0);
    size_t wbytes = (size_t)N_KINDS * SLOT;
    uint8_t *w = janas_aligned_alloc(4096, wbytes);
    size_t nbx = (MAXC + JANAS_QK - 1) / JANAS_QK;
    struct janas_block_q8k *x = malloc(NV_MAX * nbx * sizeof(*x));
    float *ya = malloc(ROWS * NV_MAX * sizeof(float)),
          *yb = malloc(ROWS * NV_MAX * sizeof(float)), xf[3 * JANAS_QK];
    struct janas_pool *pool = janas_pool_create(2, NULL);
    if (!w || !x || !ya || !yb || !pool)
        goto done;
    for (int k = 0; k < N_KINDS; k++)
        fill(&kinds[k], w + (size_t)k * SLOT, SLOT);
    for (size_t v = 0; v < NV_MAX; v++) {
        /* the activations of a row ending inside a group: its last group
           whole, past the row's end zero (model_ffn.c) */
        for (size_t i = 0; i < nbx * JANAS_QK; i++)
            xf[i] =
                i < MAXC ? (float)((int)(rnd() % 2001) - 1000) / 500.0f : 0.0f;
        janas_q8k_quantize(xf, x + v * nbx, nbx * JANAS_QK);
    }
    if (janas_gpu_import(g, w, wbytes) != 0)
        goto done;
    int was_row = g->row_on, was_mvd = g->mvd_on, was_mv = g->mv_on;
    enum { MAXW = 64 };
    struct wrong bad[MAXW];
    int n = run_all(g, pool, w, x, ya, yb, NULL, 0, bad, MAXW);
    if (n) {
        /* the families with another shader for the same products off, as
           far as one of theirs was wrong; then again */
        for (int i = 0; i < n; i++) {
            if (strstr(bad[i].shader, "_row "))
                g->row_on = 0;
            if (strstr(bad[i].shader, "_mvd "))
                g->mvd_on = 0;
            if (strstr(bad[i].shader, "q4k_mv "))
                g->mv_on = 0;
        }
        struct wrong still[MAXW];
        int m = run_all(g, pool, w, x, ya, yb, bad, n, still, MAXW);
        /* what is still wrong: the type to the CPU */
        for (int i = 0; i < m; i++)
            if (!janas_g_type_off(g, kinds[still[i].k].type) && g->n_off < 16)
                g->off_types[g->n_off++] = kinds[still[i].k].type;
        make_note(g, g->row_on || !was_row, g->mvd_on || !was_mvd,
                  g->mv_on || !was_mv);
        fprintf(stderr,
                "gpu: %d of the GPU's products differ from the CPU's on this "
                "driver (first: %s %s, %zu wide, %zu vectors); left out: "
                "%s (JANAS_GPU_SELFCHECK=0 keeps them, gpu_check tells "
                "more)\n",
                n, bad[0].shader, type_name(kinds[bad[0].k].type), bad[0].cols,
                bad[0].nv, g->check_note);
    }
    save(g, key, g->row_on || !was_row, g->mvd_on || !was_mvd,
         g->mv_on || !was_mv);
    janas_g_drop_region(g, w);
    clock_gettime(CLOCK_MONOTONIC, &t1);
    if (g->verbose || getenv("JANAS_GPU_VERBOSE"))
        fprintf(stderr, "gpu self-check: %d wrong, %.0f ms (%s)\n", n,
                (t1.tv_sec - t0.tv_sec) * 1e3 + (t1.tv_nsec - t0.tv_nsec) / 1e6,
                g->check_note[0] ? g->check_note : "all right");
done:
    janas_pool_destroy(pool);
    janas_aligned_free(w);
    free(x);
    free(ya);
    free(yb);
}

const char *janas_gpu_check_note(const struct janas_gpu *g)
{
    return g->check_note;
}

#else

const char *janas_gpu_check_note(const struct janas_gpu *g)
{
    (void)g;
    return "";
}

#endif