Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

91.4 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * test_quant.c - correctness of the quantized formats and kernels.
 *
 * Every optimised kernel must agree with the portable scalar reference; the
 * reference itself is checked against a plain float dot product over the
 * dequantized weights.
 */
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include "common/pool.h"
#include "llm/bf16.h"
#include "llm/matvec.h"
#include "llm/quant.h"
#include "llm/vmath.h"

static int failures;

#define CHECK(cond, ...)                                                       \
    do {                                                                       \
        if (!(cond)) {                                                         \
            failures++;                                                        \
            fprintf(stderr, "FAIL %s:%d: ", __FILE__, __LINE__);               \
            fprintf(stderr, __VA_ARGS__);                                      \
            fputc('\n', stderr);                                               \
        }                                                                      \
    } while (0)

/* An activation as the product of weight type t sees it: weights in
   blocks of 32 take the eight blocks of 32 (d32, qs32), the others the
   256 with one scale. */
static double act(const struct janas_block_q8k *x, int b, int i, int t)
{
    int b32 = t == JANAS_Q8_0 || t == JANAS_Q4_0 || t == JANAS_Q4_1 ||
              t == JANAS_Q5_1 || t == JANAS_IQ4_NL || t == JANAS_MXFP4 ||
              t == JANAS_Q5_0 || t == JANAS_Q2_0;
    return b32 ? (double)x[b].d32[i / 32] * x[b].qs32[i]
               : (double)x[b].d * x[b].qs[i];
}

static uint64_t rng_state = 0x9E3779B97F4A7C15ull;

static uint32_t rnd(void)
{
    rng_state ^= rng_state << 13;
    rng_state ^= rng_state >> 7;
    rng_state ^= rng_state << 17;
    return (uint32_t)(rng_state >> 32);
}

static float rndf(float lo, float hi)
{
    return lo + (hi - lo) * (float)(rnd() & 0xffffff) / (float)0x1000000;
}

static void random_q4k(struct janas_block_q4k *b, size_t n)
{
    for (size_t i = 0; i < n; i++) {
        b[i].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
        b[i].dmin = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
        for (int j = 0; j < 12; j++)
            b[i].scales[j] = (uint8_t)rnd();
        for (int j = 0; j < JANAS_QK / 2; j++)
            b[i].qs[j] = (uint8_t)rnd();
    }
}

static void test_fp16(void)
{
    const float values[] = {0.0f,     1.0f,           -2.5f,          0.1f,
                            65504.0f, 6.1035156e-05f, 5.9604645e-08f, 1234.5f};
    for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
        float v = values[i];
        float r = janas_fp16_to_fp32(janas_fp32_to_fp16(v));
        CHECK(fabsf(r - v) <= fabsf(v) * 1e-3f, "fp16 round trip %g -> %g", v,
              r);
    }
    /* every fp16 value survives fp16 -> fp32 -> fp16 exactly (NaNs aside) */
    for (uint32_t h = 0; h < 0x10000; h++) {
        if ((h & 0x7c00) == 0x7c00 && (h & 0x3ff))
            continue;
        uint16_t back = janas_fp32_to_fp16(janas_fp16_to_fp32((uint16_t)h));
        CHECK(back == h, "fp16 0x%04x -> 0x%04x", h, back);
        if (back != h)
            break;
    }
}

static void test_dot(void)
{
    enum { NB = 32, N = NB * JANAS_QK };
    struct janas_block_q4k *w = malloc(NB * sizeof(*w));
    struct janas_block_q8k *x = malloc(NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float));
    float *wf = malloc(N * sizeof(float));

    for (int trial = 0; trial < 200; trial++) {
        random_q4k(w, NB);
        for (int i = 0; i < N; i++)
            xf[i] = rndf(-1.0f, 1.0f);
        janas_q8k_quantize(xf, x, N);

        /* reference vs float product over the dequantized weights */
        janas_q4k_dequantize(w, wf, N);
        double expect = 0.0, mag = 0.0;
        for (int b = 0; b < NB; b++)
            for (int i = 0; i < JANAS_QK; i++) {
                double t = (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];
                expect += t;
                mag += fabs(t);
            }
        float ref = janas_q4k_dot_ref(w, x, NB);
        CHECK(fabs(ref - expect) <= 1e-5 * mag, "ref %g vs float %g", ref,
              expect);

        /* the same float operations: bit-identical */
        float fast = janas_q4k_dot(w, x, NB);
        CHECK(fast == ref, "%s %g vs ref %g", janas_q4k_kernel_name(), fast,
              ref);
    }
    free(w);
    free(x);
    free(xf);
    free(wf);
}

static void test_dot_q6k(void)
{
    enum { NB = 16, N = NB * JANAS_QK };
    struct janas_block_q6k *w = malloc(NB * sizeof(*w));
    struct janas_block_q8k *x = malloc(NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float));
    float *wf = malloc(N * sizeof(float));

    for (int trial = 0; trial < 200; trial++) {
        unsigned char *p = (unsigned char *)w;
        for (size_t i = 0; i < NB * sizeof(*w); i++)
            p[i] = (unsigned char)rnd();
        for (int b = 0; b < NB; b++)
            w[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
        for (int i = 0; i < N; i++)
            xf[i] = rndf(-1.0f, 1.0f);
        janas_q8k_quantize(xf, x, N);

        janas_q6k_dequantize(w, wf, N);
        double expect = 0.0, mag = 0.0;
        for (int b = 0; b < NB; b++)
            for (int i = 0; i < JANAS_QK; i++) {
                double t = (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];
                expect += t;
                mag += fabs(t);
            }
        float ref = janas_q6k_dot_ref(w, x, NB);
        CHECK(fabs(ref - expect) <= 1e-5 * mag, "q6k ref %g vs float %g", ref,
              expect);
        float fast = janas_q6k_dot(w, x, NB);
        CHECK(fast == ref, "q6k fast %g vs ref %g", fast, ref);
    }
    free(w);
    free(x);
    free(xf);
    free(wf);
}

/*
 * Q6_K cut into three two-bit planes: the split and its inverse lose nothing,
 * the fast kernels agree with the scalar reference at every level, with all
 * three planes the result is the Q6_K one bit for bit, and with fewer planes
 * every weight sits within half a step of the centre it stands for.
 */
static void test_q6k_planes(void)
{
    enum { NB = 16, NV = 9, N = NB * JANAS_QK };
    struct janas_block_q6k *w = malloc(NB * sizeof(*w));
    struct janas_block_q6k *back = malloc(NB * sizeof(*back));
    struct janas_block_q6kp *base = malloc(NB * sizeof(*base));
    uint8_t *p1 = malloc(NB * JANAS_Q6KP_PLANE);
    uint8_t *p0 = malloc(NB * JANAS_Q6KP_PLANE);
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float));
    float *wf = malloc(N * sizeof(float));
    float *pf = malloc(N * sizeof(float));

    for (int trial = 0; trial < 50; trial++) {
        unsigned char *p = (unsigned char *)w;
        for (size_t i = 0; i < NB * sizeof(*w); i++)
            p[i] = (unsigned char)rnd();
        for (int b = 0; b < NB; b++)
            w[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }

        janas_q6k_to_planes(w, NB, base, p1, p0);
        janas_q6kp_to_q6k(base, p1, p0, NB, back);
        CHECK(!memcmp(w, back, NB * sizeof(*w)), "planes: round trip differs");

        float r6 = janas_q6k_dot_ref(w, x, NB);
        CHECK(janas_q6kp_dot_ref(base, p1, p0, x, NB) == r6,
              "planes ref level 3: %g vs q6k %g",
              janas_q6kp_dot_ref(base, p1, p0, x, NB), r6);
        CHECK(janas_q6kp_dot(base, p1, p0, x, NB) == janas_q6k_dot(w, x, NB),
              "planes fast level 3 differs from q6k");
        for (int level = 1; level <= 3; level++) {
            const uint8_t *a1 = level >= 2 ? p1 : NULL;
            const uint8_t *a0 = level >= 3 ? p0 : NULL;
            float ref = janas_q6kp_dot_ref(base, a1, a0, x, NB);
            CHECK(janas_q6kp_dot(base, a1, a0, x, NB) == ref,
                  "planes level %d: fast %g vs ref %g", level,
                  janas_q6kp_dot(base, a1, a0, x, NB), ref);
            float out[NV];
            for (int nv = 1; nv <= NV; nv++) {
                janas_q6kp_dot_multi(base, a1, a0, x, NB, (size_t)nv, NB, out,
                                     1);
                for (int v = 0; v < nv; v++) {
                    float r = janas_q6kp_dot_ref(base, a1, a0, x + v * NB, NB);
                    CHECK(out[v] == r, "planes level %d multi nv %d v %d",
                          level, nv, v);
                }
            }
        }

        janas_q6k_dequantize(w, wf, N);
        for (int level = 1; level <= 2; level++) {
            janas_q6kp_dequantize(base, level >= 2 ? p1 : NULL, NULL, pf, N);
            float half = level == 2 ? 1.5f : 7.5f;
            for (int b = 0; b < NB; b++) {
                float d = janas_fp16_to_fp32(w[b].d);
                for (int i = 0; i < JANAS_QK; i++) {
                    int k = b * JANAS_QK + i;
                    float step = fabsf(d * w[b].scales[i / 16]);
                    CHECK(fabsf(pf[k] - wf[k]) <= half * step * 1.001f,
                          "planes level %d weight %d: %g vs %g (step %g)",
                          level, k, pf[k], wf[k], step);
                }
            }
        }
    }
    free(w);
    free(back);
    free(base);
    free(p1);
    free(p0);
    free(x);
    free(xf);
    free(wf);
    free(pf);
}

/*
 * Q6_K and Q6_K_P as tiles of eight rows (janas_q6k_tile_rows): 11 rows, 9
 * blocks (past the eight transposed at a time), 1 to 65 vectors (past the
 * 64 tokens at a time), Q6_K_P at every level; each result janas_q6k_dot's
 * or janas_q6kp_dot's to the bit, and the rows done a multiple of eight.
 */
static void test_q6k_tiles(void)
{
    enum { ROWS = 11, NB = 9, NV = 65, N = NB * JANAS_QK };
    size_t pb = NB * JANAS_Q6KP_PLANE;
    struct janas_block_q6k *w = malloc(ROWS * NB * sizeof(*w));
    struct janas_block_q6kp *base = malloc(ROWS * NB * sizeof(*base));
    uint8_t *p1 = malloc(ROWS * pb), *p0 = malloc(ROWS * pb);
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float));
    float *out = malloc(ROWS * NV * sizeof(float));
    unsigned char *p = (unsigned char *)w;
    for (size_t i = 0; i < ROWS * NB * sizeof(*w); i++)
        p[i] = (unsigned char)rnd();
    for (int b = 0; b < ROWS * NB; b++)
        w[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
    for (int r = 0; r < ROWS; r++)
        janas_q6k_to_planes(w + r * NB, NB, base + r * NB, p1 + r * pb,
                            p0 + r * pb);
    for (int v = 0; v < NV; v++) {
        for (int i = 0; i < N; i++)
            xf[i] = rndf(-1.0f, 1.0f);
        janas_q8k_quantize(xf, x + v * NB, N);
    }
    for (int level = 0; level <= 3; level++) {
        const uint8_t *a1 = level >= 2 ? p1 : NULL;
        const uint8_t *a0 = level >= 3 ? p0 : NULL;
        int t = level ? JANAS_Q6_K_P : JANAS_Q6_K;
        const void *wr = level ? (const void *)base : (const void *)w;
        size_t rb = level ? NB * sizeof(*base) : NB * sizeof(*w);
        for (int nv = 1; nv <= NV; nv += nv < 9 ? 1 : 8) {
            for (int i = 0; i < ROWS * NV; i++)
                out[i] = -12345.0f;
            size_t done = janas_q6k_tile_rows(t, wr, rb, a1, a0, pb, x, NB,
                                              (size_t)nv, NB, ROWS, out, ROWS);
            CHECK(done == (nv >= 12 ? 8u : 0u), "q6k tiles: %zu rows", done);
            for (size_t r = 0; r < done; r++)
                for (int v = 0; v < nv; v++) {
                    float one = level
                                    ? janas_q6kp_dot(base + r * NB,
                                                     a1 ? a1 + r * pb : NULL,
                                                     a0 ? a0 + r * pb : NULL,
                                                     x + v * NB, NB)
                                    : janas_q6k_dot(w + r * NB, x + v * NB, NB);
                    CHECK(out[r + ROWS * v] == one,
                          "q6k tiles level %d nv %d r %zu v %d: %g vs %g",
                          level, nv, r, v, out[r + ROWS * v], one);
                }
        }
    }
    free(w);
    free(base);
    free(p1);
    free(p0);
    free(x);
    free(xf);
    free(out);
}

/* janas_q6kpn_axpy at every level: with h = 1 on zeros the very floats
   janas_q6kp_dequantize gives, whatever planes lie below the level; with
   any h on any y, y plus h times them. */
static void test_q6kpn_axpy(void)
{
    enum { NB = 16, N = NB * JANAS_QK };
    struct janas_block_q6k *w = malloc(NB * sizeof(*w));
    struct janas_block_q6kp *base = malloc(NB * sizeof(*base));
    uint8_t *p1 = malloc(NB * JANAS_Q6KP_PLANE);
    uint8_t *p0 = malloc(NB * JANAS_Q6KP_PLANE);
    float *ref = malloc(N * sizeof(float)), *y = malloc(N * sizeof(float));
    float *y0 = malloc(N * sizeof(float));

    for (int trial = 0; trial < 20; trial++) {
        unsigned char *p = (unsigned char *)w;
        for (size_t i = 0; i < NB * sizeof(*w); i++)
            p[i] = (unsigned char)rnd();
        for (int b = 0; b < NB; b++)
            w[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
        janas_q6k_to_planes(w, NB, base, p1, p0);
        for (int level = 0; level <= 3; level++) {
            const uint8_t *a1 = level >= 2 ? p1 : NULL;
            const uint8_t *a0 = level >= 3 ? p0 : NULL;
            if (level)
                janas_q6kp_dequantize(base, a1, a0, ref, N);
            for (int i = 0; i < N; i++) {
                if (!level)
                    ref[i] = 0.0f;
                y[i] = 0.0f;
            }
            janas_q6kpn_axpy(base, p1, p0, level, 1.0f, y, N);
            int bad = 0;
            for (int i = 0; i < N; i++)
                bad += y[i] != ref[i];
            CHECK(!bad, "q6kpn axpy level %d, h = 1: %d of %d floats differ",
                  level, bad, N);
            float h = rndf(-2.0f, 2.0f);
            for (int i = 0; i < N; i++)
                y0[i] = y[i] = rndf(-1.0f, 1.0f);
            janas_q6kpn_axpy(base, a1, a0, level, h, y, N);
            bad = 0;
            for (int i = 0; i < N; i++) {
                float want = y0[i] + h * ref[i];
                bad += fabsf(y[i] - want) > 1e-6f * (1.0f + fabsf(want));
            }
            CHECK(!bad, "q6kpn axpy level %d, h = %g: %d of %d floats off",
                  level, h, bad, N);
            /* the kernel in use against the reference, bit for bit */
            float *yr = ref;
            for (int i = 0; i < N; i++)
                yr[i] = y0[i] = y[i] = rndf(-1.0f, 1.0f);
            janas_q6kpn_axpy(base, a1, a0, level, h, y, N);
            janas_q6kpn_axpy_ref(base, a1, a0, level, h, yr, N);
            CHECK(!memcmp(y, yr, N * sizeof(float)),
                  "q6kpn axpy level %d: kernel and reference differ", level);
        }
    }
    free(w);
    free(base);
    free(p1);
    free(p0);
    free(ref);
    free(y);
    free(y0);
}

/* janas_q6kpn_chunk: the kernel in use against the reference, bit for bit,
   on chunks of 1 to 128 neurons at random levels (JANAS_KERNELS=avx2 tries
   the AVX2 one on a machine with AVX-VNNI); and the reference against the
   float axpy, the same products with factors of eight bits. */
static void test_q6kpn_chunk(void)
{
    enum { NN = 160, NB = 4 };
    struct janas_block_q6k *w = malloc(NN * NB * sizeof(*w));
    struct janas_block_q6kp *base = malloc(NN * NB * sizeof(*base));
    uint8_t *p1 = malloc(NN * NB * JANAS_Q6KP_PLANE);
    uint8_t *p0 = malloc(NN * NB * JANAS_Q6KP_PLANE);
    unsigned char *p = (unsigned char *)w;
    for (size_t i = 0; i < NN * NB * sizeof(*w); i++)
        p[i] = (unsigned char)rnd();
    for (int b = 0; b < NN * NB; b++)
        w[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
    janas_q6k_to_planes(w, NN * NB, base, p1, p0);
    double worst = 0;
    for (int trial = 0; trial < 60; trial++) {
        uint32_t js[JANAS_PN_CHUNK], perm[NN];
        uint8_t lv[JANAS_PN_CHUNK];
        float h[JANAS_PN_CHUNK], y0[JANAS_QK], y1[JANAS_QK], y2[JANAS_QK],
            ya[JANAS_QK];
        uint32_t n = trial < 3   ? (uint32_t)trial + 1
                     : trial < 6 ? JANAS_PN_CHUNK - (uint32_t)trial + 3
                                 : 1 + rnd() % JANAS_PN_CHUNK;
        for (uint32_t i = 0; i < NN; i++)
            perm[i] = i;
        for (uint32_t i = 0; i < n; i++) { /* distinct neurons */
            uint32_t k = i + rnd() % (NN - i), t = perm[i];
            perm[i] = perm[k];
            perm[k] = t;
            js[i] = perm[i];
            lv[i] = (uint8_t)(1 + rnd() % 3);
            float g = rndf(-1, 1);
            h[i] = trial % 2 ? g * g * g * 4.0f : rndf(-2.0f, 2.0f);
        }
        size_t b = rnd() % NB;
        for (int v = 0; v < JANAS_QK; v++)
            ya[v] = y0[v] = y1[v] = y2[v] = rndf(-1.0f, 1.0f);
        janas_q6kpn_chunk(base, p1, p0, NN, js, lv, h, n, b, y1);
        janas_q6kpn_chunk_ref(base, p1, p0, NN, js, lv, h, n, b, y2);
        CHECK(!memcmp(y1, y2, sizeof(y1)),
              "q6kpn chunk of %u: kernel and reference differ", n);
        for (uint32_t i = 0; i < n; i++) {
            size_t at = b * NN + js[i]; /* by block of outputs */
            janas_q6kpn_axpy_ref(base + at, p1 + at * JANAS_Q6KP_PLANE,
                                 p0 + at * JANAS_Q6KP_PLANE, lv[i], h[i], ya,
                                 JANAS_QK);
        }
        double e = 0, sg = 0;
        for (int v = 0; v < JANAS_QK; v++) {
            e += ((double)y2[v] - ya[v]) * (y2[v] - ya[v]);
            sg += ((double)ya[v] - y0[v]) * (ya[v] - y0[v]);
        }
        double rel = sg > 0 ? sqrt(e / sg) : 0;
        if (rel > worst)
            worst = rel;
        CHECK(rel < 0.02, "q6kpn chunk of %u: %.4f off the float products", n,
              rel);
    }
    printf("q6kpn chunk: factors of eight bits, at most %.4f off the float "
           "products\n",
           worst);
    free(w);
    free(base);
    free(p1);
    free(p0);
}

static void test_matvec(void)
{
    enum { ROWS = 1000, COLS = 2048, NB = COLS / JANAS_QK };
    struct janas_block_q4k *w = malloc((size_t)ROWS * NB * sizeof(*w));
    struct janas_block_q8k *x = malloc(NB * sizeof(*x));
    float xf[COLS], y[ROWS];

    random_q4k(w, (size_t)ROWS * NB);
    for (int i = 0; i < COLS; i++)
        xf[i] = rndf(-1.0f, 1.0f);
    janas_q8k_quantize(xf, x, COLS);

    for (int threads = 1; threads <= 4; threads *= 2) {
        struct janas_pool *pool = janas_pool_create(threads, NULL);
        CHECK(pool != NULL, "pool of %d threads", threads);
        if (!pool)
            continue;
        for (int rep = 0; rep < 20; rep++) {
            memset(y, 0, sizeof(y));
            janas_matvec_q4k(pool, w, ROWS, COLS, x, y);
            for (int r = 0; r < ROWS; r++) {
                float ref = janas_q4k_dot(w + (size_t)r * NB, x, NB);
                CHECK(y[r] == ref, "matvec threads %d row %d: %g vs %g",
                      threads, r, y[r], ref);
                if (y[r] != ref)
                    break;
            }
        }
        janas_pool_destroy(pool);
    }
    free(w);
    free(x);
}

static void test_matvec_group(void)
{
    /* expert-like shapes, one empty task, different inputs per task */
    const size_t rows[] = {512, 0, 512, 2048, 37, 256};
    const size_t cols[] = {2048, 2048, 2048, 512, 768, 256};
    enum { N = 6 };
    struct janas_matvec_task tasks[N];
    for (int i = 0; i < N; i++) {
        size_t nb = cols[i] / JANAS_QK;
        struct janas_block_q4k *w = malloc((rows[i] * nb + 1) * sizeof(*w));
        struct janas_block_q8k *x = malloc(nb * sizeof(*x));
        float *xf = malloc(cols[i] * sizeof(float));
        random_q4k(w, rows[i] * nb);
        for (size_t j = 0; j < cols[i]; j++)
            xf[j] = rndf(-1.0f, 1.0f);
        janas_q8k_quantize(xf, x, cols[i]);
        free(xf);
        tasks[i] = (struct janas_matvec_task){
            .type = JANAS_Q4_K,
            .w = w,
            .x = x,
            .y = malloc((rows[i] + 1) * sizeof(float)),
            .rows = rows[i],
            .cols = cols[i]};
    }
    struct janas_pool *pool = janas_pool_create(3, NULL);
    CHECK(pool != NULL, "pool");
    for (int rep = 0; pool && rep < 10; rep++) {
        janas_matvec_q4k_group(pool, tasks, N);
        for (int i = 0; i < N; i++) {
            size_t nb = cols[i] / JANAS_QK;
            for (size_t r = 0; r < rows[i]; r++) {
                const struct janas_block_q4k *tw = tasks[i].w;
                float ref = janas_q4k_dot(tw + r * nb, tasks[i].x, nb);
                CHECK(tasks[i].y[r] == ref, "group task %d row %zu: %g vs %g",
                      i, r, tasks[i].y[r], ref);
                if (tasks[i].y[r] != ref)
                    break;
            }
        }
    }
    janas_pool_destroy(pool);
    for (int i = 0; i < N; i++) {
        free((void *)tasks[i].w);
        free((void *)tasks[i].x);
        free(tasks[i].y);
    }
}

static void test_matvec_multi(void)
{
    enum { ROWS = 300, COLS = 512, NB = COLS / JANAS_QK, NV = 5 };
    struct janas_block_q4k *w = malloc((size_t)ROWS * NB * sizeof(*w));
    struct janas_block_q8k *x = malloc((size_t)NV * NB * sizeof(*x));
    float xf[COLS], y[NV * ROWS];
    random_q4k(w, (size_t)ROWS * NB);
    for (int v = 0; v < NV; v++) {
        for (int i = 0; i < COLS; i++)
            xf[i] = rndf(-1.0f, 1.0f);
        janas_q8k_quantize(xf, x + (size_t)v * NB, COLS);
    }
    struct janas_pool *pool = janas_pool_create(3, NULL);
    struct janas_matvec_task t = {.type = JANAS_Q4_K,
                                  .w = w,
                                  .x = x,
                                  .y = y,
                                  .rows = ROWS,
                                  .cols = COLS,
                                  .n_vec = NV};
    janas_matvec_q4k_group(pool, &t, 1);
    for (int v = 0; v < NV; v++)
        for (int r = 0; r < ROWS; r++) {
            float ref =
                janas_q4k_dot(w + (size_t)r * NB, x + (size_t)v * NB, NB);
            CHECK(y[v * ROWS + r] == ref, "multi v %d row %d: %g vs %g", v, r,
                  y[v * ROWS + r], ref);
        }
    janas_pool_destroy(pool);
    free(w);
    free(x);
}

/*
 * Q4_K quantization: round trip on Gaussian-like and heavy-tailed weights.
 * The error relative to the weights' spread must stay near what 4 bits with
 * 6-bit sub-block scales allow, and below a plain min/max rounding - also
 * for the fast way, which takes each sub-block's range as it is.
 */
static void test_quantize_q4k(void)
{
    enum { N = 256 * 64 };
    float *x = malloc(N * sizeof(float)), *y = malloc(N * sizeof(float));
    struct janas_block_q4k *q = malloc(N / JANAS_QK * sizeof(*q));
    for (int k2 = 0; k2 < 4; k2++) {
        int kind = k2 & 1, fast = k2 >> 1;
        double var = 0, err = 0, naive = 0;
        for (int i = 0; i < N; i++) {
            /* sum of uniforms ~ Gaussian; its cube is heavy-tailed */
            float g = rndf(-1, 1) + rndf(-1, 1) + rndf(-1, 1) + rndf(-1, 1);
            x[i] = 0.02f * (kind ? g * g * g : g);
            var += (double)x[i] * x[i];
        }
        if (fast)
            janas_q4k_quantize_fast(x, q, N);
        else
            janas_q4k_quantize(x, q, N);
        janas_q4k_dequantize(q, y, N);
        for (int i = 0; i < N; i++)
            err += ((double)y[i] - x[i]) * (y[i] - x[i]);
        for (int s = 0; s < N / 32; s++) { /* min/max rounding, float scales */
            float lo = x[32 * s], hi = lo;
            for (int k = 0; k < 32; k++) {
                lo = fminf(lo, x[32 * s + k]);
                hi = fmaxf(hi, x[32 * s + k]);
            }
            float st = (hi - lo) / 15;
            for (int k = 0; k < 32; k++) {
                float v =
                    st > 0 ? lo + st * lrintf((x[32 * s + k] - lo) / st) : lo;
                naive += ((double)v - x[32 * s + k]) * (v - x[32 * s + k]);
            }
        }
        double rel = sqrt(err / var), rel_naive = sqrt(naive / var);
        printf("q4k quantize%s, %s weights: relative rms error %.4f (min/max "
               "rounding %.4f)\n",
               fast ? " fast" : "", kind ? "heavy-tailed" : "Gaussian", rel,
               rel_naive);
        CHECK(rel < (kind ? 0.20 : 0.11) && rel < rel_naive,
              "q4k quantize %d%s: relative error %.4f, naive %.4f", kind,
              fast ? " fast" : "", rel, rel_naive);
    }
    free(x);
    free(y);
    free(q);
}

/* janas_q6k_quantize: every weight read back as the integer nearest to it
   under the scales the block keeps (so its bits sit where Q6_K reads them),
   and less error than the plain way: each group of 16 at its largest
   magnitude over 31, those scales then to the block's eight bits. */
static void test_quantize_q6k(void)
{
    enum { N = 256 * 64 };
    float *x = malloc(N * sizeof(float)), *y = malloc(N * sizeof(float));
    struct janas_block_q6k *q = malloc(N / JANAS_QK * sizeof(*q));
    for (int kind = 0; kind < 2; kind++) {
        double var = 0, err = 0, plain = 0, plain_f = 0;
        for (int i = 0; i < N; i++) {
            /* sum of uniforms ~ Gaussian; its cube is heavy-tailed */
            float g = rndf(-1, 1) + rndf(-1, 1) + rndf(-1, 1) + rndf(-1, 1);
            x[i] = 0.02f * (kind ? g * g * g : g);
            var += (double)x[i] * x[i];
        }
        janas_q6k_quantize(x, q, N);
        janas_q6k_dequantize(q, y, N);
        int far = 0;
        for (int i = 0; i < N; i++) {
            const struct janas_block_q6k *b = &q[i / JANAS_QK];
            float a = janas_fp16_to_fp32(b->d) * b->scales[i % JANAS_QK / 16];
            int k = a != 0.0f ? (int)lrintf(x[i] / a) : 0;
            k = k < -32 ? -32 : k > 31 ? 31 : k;
            far += y[i] != a * k;
            err += ((double)y[i] - x[i]) * (y[i] - x[i]);
        }
        CHECK(!far, "q6k quantize %d: %d weights not the nearest", kind, far);
        for (int b = 0; b < N / JANAS_QK; b++) {
            const float *xb = x + b * JANAS_QK;
            float a[16], amax = 0;
            for (int s = 0; s < 16; s++) {
                a[s] = 0;
                for (int k = 0; k < 16; k++)
                    a[s] = fmaxf(a[s], fabsf(xb[16 * s + k]) / 31);
                amax = fmaxf(amax, a[s]);
            }
            float d = janas_fp16_to_fp32(janas_fp32_to_fp16(amax / 127));
            for (int v = 0; v < JANAS_QK; v++) {
                float af = a[v / 16], a8 = d * lrintf(af / d);
                int kf = af > 0 ? (int)lrintf(xb[v] / af) : 0;
                int k8 = a8 > 0 ? (int)lrintf(xb[v] / a8) : 0;
                kf = kf < -32 ? -32 : kf > 31 ? 31 : kf;
                k8 = k8 < -32 ? -32 : k8 > 31 ? 31 : k8;
                plain_f += ((double)af * kf - xb[v]) * (af * kf - xb[v]);
                plain += ((double)a8 * k8 - xb[v]) * (a8 * k8 - xb[v]);
            }
        }
        double rel = sqrt(err / var), rel_plain = sqrt(plain / var);
        printf("q6k quantize, %s weights: relative rms error %.5f (plain "
               "%.5f, with float scales %.5f)\n",
               kind ? "heavy-tailed" : "Gaussian", rel, rel_plain,
               sqrt(plain_f / var));
        CHECK(rel < rel_plain,
              "q6k quantize %d: relative error %.5f, plain "
              "%.5f",
              kind, rel, rel_plain);
    }
    free(x);
    free(y);
    free(q);
}

/* janas_dot_multi on 1 to 17 vectors, both types: equal to the scalar
   reference for every vector (the kernels group vectors by eight). */
static void test_dot_multi(void)
{
    enum { NB = 8, NV = 17, N = NB * JANAS_QK };
    struct janas_block_q4k *w4 = malloc(NB * sizeof(*w4));
    struct janas_block_q6k *w6 = malloc(NB * sizeof(*w6));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float));
    random_q4k(w4, NB);
    unsigned char *p = (unsigned char *)w6;
    for (size_t i = 0; i < NB * sizeof(*w6); i++)
        p[i] = (unsigned char)rnd();
    for (int b = 0; b < NB; b++)
        w6[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
    for (int v = 0; v < NV; v++) {
        for (int i = 0; i < N; i++)
            xf[i] = rndf(-1.0f, 1.0f);
        janas_q8k_quantize(xf, x + v * NB, N);
    }
    for (int nv = 1; nv <= NV; nv++) {
        float out[NV];
        janas_dot_multi(JANAS_Q4_K, w4, x, NB, (size_t)nv, NB, out, 1);
        for (int v = 0; v < nv; v++) {
            float ref = janas_q4k_dot_ref(w4, x + v * NB, NB);
            CHECK(out[v] == ref, "q4k multi nv %d v %d: %g vs %g", nv, v,
                  out[v], ref);
        }
        janas_dot_multi(JANAS_Q6_K, w6, x, NB, (size_t)nv, NB, out, 1);
        for (int v = 0; v < nv; v++) {
            float ref = janas_q6k_dot_ref(w6, x + v * NB, NB);
            CHECK(out[v] == ref, "q6k multi nv %d v %d: %g vs %g", nv, v,
                  out[v], ref);
        }
    }
    free(w4);
    free(w6);
    free(x);
    free(xf);
}

#if defined(__x86_64__)
__attribute__((target("avx2,fma"))) static void
det8(const float *g, const float *u, float *h, int n)
{
    for (int i = 0; i < n; i += 8)
        _mm256_storeu_ps(h + i, janas_silu_mul_det8(_mm256_loadu_ps(g + i),
                                                    _mm256_loadu_ps(u + i)));
}
#endif

/* The deterministic functions: scalar and AVX2 give the same bits; the
   reciprocal is within an ulp; the deterministic quantization is within one
   step of the plain one. */
static void test_det(void)
{
    float g[1024], u[1024], a[1024], b[1024];
    int bad = 0, compared = 0;
    for (int round = 0; round < 1000; round++) {
        for (int i = 0; i < 1024; i++) {
            /* wide range, and the clamp edges */
            g[i] = rndf(-1.0f, 1.0f) * (i % 7 == 0 ? 120.0f : 12.0f);
            u[i] = rndf(-4.0f, 4.0f);
        }
        g[0] = -30.0f, g[1] = -30.5f, g[2] = 88.0f, g[3] = -88.0f;
        g[4] = 0.0f, g[5] = -0.0f, g[6] = 1e-30f, g[7] = 100.0f;
        for (int i = 0; i < 1024; i++)
            a[i] = janas_silu_mul_det(g[i], u[i]);
#if defined(__x86_64__)
        if (__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma")) {
            det8(g, u, b, 1024);
            for (int i = 0; i < 1024; i++)
                bad += memcmp(&a[i], &b[i], 4) != 0;
            compared += 1024;
        }
#endif
    }
    /* without AVX2 and FMA there is nothing to compare it with: say so,
       rather than pass as if 1000 rounds had agreed (issue #26) */
    if (compared)
        printf("silu_mul_det: scalar against AVX2 on %d values\n", compared);
    else
        printf("silu_mul_det: no AVX2 with FMA here, scalar not compared\n");
    CHECK(bad == 0, "silu_mul_det: %d scalar/AVX2 differences of %d", bad,
          compared);
    double worst = 0;
    for (int i = 0; i < 100000; i++) {
        float d = ldexpf(rndf(1.0f, 2.0f), (int)(rnd() % 200) - 100);
        double r = janas_recip_det(d), e = fabs(r * d - 1.0);
        if (e > worst)
            worst = e;
    }
    CHECK(worst < 1.2e-7, "recip_det: worst relative error %g", worst);
    enum { N = 1024 };
    float x[N];
    struct janas_block_q8k qa[N / JANAS_QK], qb[N / JANAS_QK];
    for (int i = 0; i < N; i++)
        x[i] = rndf(-3.0f, 3.0f);
    janas_q8k_quantize(x, qa, N);
    janas_q8k_quantize_det(x, qb, N);
    int far = 0;
    for (int b = 0; b < N / JANAS_QK; b++)
        for (int i = 0; i < JANAS_QK; i++)
            far += abs(qa[b].qs[i] - qb[b].qs[i]) > 1;
    CHECK(far == 0, "quantize_det: %d values off by more than one", far);
}

/* Q5_K and Q8_0: the reference against a float product over the
   dequantized weights, the kernels (single and 1-17 vectors) equal to it. */
static void test_dot_q5k_q8_0(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    struct janas_block_q5k *w5 = malloc(NB * sizeof(*w5));
    struct janas_block_q8_0 *w8 = malloc(NB * 8 * sizeof(*w8));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float));
    for (int trial = 0; trial < 100; trial++) {
        unsigned char *p5 = (unsigned char *)w5, *p8 = (unsigned char *)w8;
        for (size_t i = 0; i < NB * sizeof(*w5); i++)
            p5[i] = (unsigned char)rnd();
        for (size_t i = 0; i < NB * 8 * sizeof(*w8); i++)
            p8[i] = (unsigned char)rnd();
        for (int b = 0; b < NB; b++) {
            w5[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
            w5[b].dmin = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
        }
        for (int b = 0; b < NB * 8; b++)
            w8[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }
        for (int type = 0; type < 2; type++) {
            const void *w = type ? (const void *)w8 : (const void *)w5;
            int t = type ? JANAS_Q8_0 : JANAS_Q5_K;
            if (type)
                janas_q8_0_dequantize(w8, wf, N);
            else
                janas_q5k_dequantize(w5, wf, N);
            double expect = 0.0, mag = 0.0;
            for (int b = 0; b < NB; b++)
                for (int i = 0; i < JANAS_QK; i++) {
                    double e = (double)wf[b * JANAS_QK + i] * act(x, b, i, t);
                    expect += e;
                    mag += fabs(e);
                }
            float ref = type ? janas_q8_0_dot_ref(w8, x, NB)
                             : janas_q5k_dot_ref(w5, x, NB);
            CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "%s ref %g vs %g",
                  type ? "q8_0" : "q5k", ref, expect);
            float fast = janas_dot(t, w, x, NB);
            CHECK(fast == ref, "%s fast %g vs ref %g", type ? "q8_0" : "q5k",
                  fast, ref);
            for (int nv = 1; nv <= NV; nv++) {
                float out[NV];
                janas_dot_multi(t, w, x, NB, (size_t)nv, NB, out, 1);
                for (int v = 0; v < nv; v++) {
                    float r = type ? janas_q8_0_dot_ref(w8, x + v * NB, NB)
                                   : janas_q5k_dot_ref(w5, x + v * NB, NB);
                    CHECK(out[v] == r, "%s multi nv %d v %d: %g vs %g",
                          type ? "q8_0" : "q5k", nv, v, out[v], r);
                }
            }
        }
    }
    free(w5);
    free(w8);
    free(x);
    free(xf);
    free(wf);
}

/* IQ4_NL and IQ4_XS: as above - the reference against a float product
   over the dequantized weights, the kernels (single and 1-17 vectors) equal
   to it to the bit. */
static void test_dot_iq4(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    struct janas_block_iq4nl *wn = malloc(NB * 8 * sizeof(*wn));
    struct janas_block_iq4xs *wx = malloc(NB * sizeof(*wx));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float));
    for (int trial = 0; trial < 100; trial++) {
        unsigned char *pn = (unsigned char *)wn, *px = (unsigned char *)wx;
        for (size_t i = 0; i < NB * 8 * sizeof(*wn); i++)
            pn[i] = (unsigned char)rnd();
        for (size_t i = 0; i < NB * sizeof(*wx); i++)
            px[i] = (unsigned char)rnd();
        for (int b = 0; b < NB * 8; b++)
            wn[b].d = janas_fp32_to_fp16(rndf(-0.01f, 0.01f));
        for (int b = 0; b < NB; b++)
            wx[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }
        for (int xs4 = 0; xs4 < 2; xs4++) {
            const void *w = xs4 ? (const void *)wx : (const void *)wn;
            int t = xs4 ? JANAS_IQ4_XS : JANAS_IQ4_NL;
            const char *name = xs4 ? "iq4_xs" : "iq4_nl";
            if (xs4)
                janas_iq4xs_dequantize(wx, wf, N);
            else
                janas_iq4nl_dequantize(wn, wf, N);
            double expect = 0.0, mag = 0.0;
            for (int b = 0; b < NB; b++)
                for (int i = 0; i < JANAS_QK; i++) {
                    double e = (double)wf[b * JANAS_QK + i] * act(x, b, i, t);
                    expect += e;
                    mag += fabs(e);
                }
            float ref = xs4 ? janas_iq4xs_dot_ref(wx, x, NB)
                            : janas_iq4nl_dot_ref(wn, x, NB);
            CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "%s ref %g vs %g",
                  name, ref, expect);
            float fast = janas_dot(t, w, x, NB);
            CHECK(fast == ref, "%s fast %g vs ref %g", name, fast, ref);
            for (int nv = 1; nv <= NV; nv++) {
                float out[NV];
                janas_dot_multi(t, w, x, NB, (size_t)nv, NB, out, 1);
                for (int v = 0; v < nv; v++) {
                    float r = xs4 ? janas_iq4xs_dot_ref(wx, x + v * NB, NB)
                                  : janas_iq4nl_dot_ref(wn, x + v * NB, NB);
                    CHECK(out[v] == r, "%s multi nv %d v %d: %g vs %g", name,
                          nv, v, out[v], r);
                }
            }
        }
    }
    /* the levels and a scale by hand: an index of 0 is -127, 15 is 113,
       and group 1's scale takes its high bits from bits 2-3 of scales_h */
    struct janas_block_iq4xs h = {0};
    h.d = janas_fp32_to_fp16(1.0f);
    h.scales_l[0] = 0x50;  /* group 1: low bits 5 */
    h.scales_h = 0x3 << 2; /* group 1: high bits 3 -> ls 53, scale 21 */
    h.qs[16] = 0xf0;       /* group 1, weights 0 and 16: levels 0 and 15 */
    float y[JANAS_QK];
    janas_iq4xs_dequantize(&h, y, JANAS_QK);
    CHECK(y[32] == 21.0f * -127 && y[48] == 21.0f * 113,
          "iq4_xs by hand: %g %g", y[32], y[48]);
    free(wn);
    free(wx);
    free(x);
    free(xf);
    free(wf);
}

/* Q4_0 and Q5_1: the reference against a float product over the
   dequantized weights (Q5_1's minimum times the activations' sums
   included), the kernels on 1-17 vectors equal to it to the bit. */
/*
 * MXFP4: the reference against a float product over the dequantized
 * weights, the kernels (single, 1-17 vectors, rows) equal to it to the bit,
 * the scale exactly 2^(e - 128) for every byte, and the same bits as its
 * exact rewrite into Q8_0 (janas_mxfp4_to_q8_0, gguf2jns's for resident
 * tensors), whose products take the activations the same way.
 */
static void test_dot_mxfp4(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    for (int e = 0; e < 256; e++)
        CHECK(janas_e8m0_half((uint8_t)e) == ldexpf(1.0f, e - 128),
              "e8m0 %d: %g", e, janas_e8m0_half((uint8_t)e));
    struct janas_block_mxfp4 *w = malloc(NB * 8 * sizeof(*w));
    struct janas_block_q8_0 *w8 = malloc(NB * 8 * sizeof(*w8));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float));
    float rows[4];
    for (int trial = 0; trial < 100; trial++) {
        unsigned char *pw = (unsigned char *)w;
        for (size_t i = 0; i < NB * 8 * sizeof(*w); i++)
            pw[i] = (unsigned char)rnd();
        /* the powers of two of real weights (2^-10 to 2^3), and in some
           trials the subnormal and the largest ones as well */
        for (int b = 0; b < NB * 8; b++)
            w[b].e = (uint8_t)(118 + rnd() % 14);
        if (trial % 4 == 3) {
            w[0].e = 0;
            w[1].e = 1;
            w[2].e = 2;
            w[3].e = 254;
        }
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }
        janas_mxfp4_dequantize(w, wf, N);
        double expect = 0.0, mag = 0.0;
        for (int b = 0; b < NB; b++)
            for (int i = 0; i < JANAS_QK; i++) {
                double p =
                    (double)wf[b * JANAS_QK + i] * act(x, b, i, JANAS_MXFP4);
                expect += p;
                mag += fabs(p);
            }
        float ref = janas_mxfp4_dot_ref(w, x, NB);
        if (trial % 4 != 3) /* 2^126 times the rest: no float to compare */
            CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "mxfp4 ref %g vs %g",
                  ref, expect);
        float fast = janas_dot(JANAS_MXFP4, w, x, NB);
        CHECK(fast == ref || (isnan(fast) && isnan(ref)),
              "mxfp4 fast %g vs ref %g", fast, ref);
        for (int nv = 1; nv <= NV; nv++) {
            float out[NV];
            janas_dot_multi(JANAS_MXFP4, w, x, NB, (size_t)nv, NB, out, 1);
            for (int v = 0; v < nv; v++) {
                float r = janas_mxfp4_dot_ref(w, x + v * NB, NB);
                CHECK(out[v] == r || (isnan(out[v]) && isnan(r)),
                      "mxfp4 multi nv %d v %d: %g vs %g", nv, v, out[v], r);
            }
        }
        /* four rows of two super-blocks each, the matrix read row by row */
        janas_dot_rows(JANAS_MXFP4, w, 2 * 8 * sizeof(*w), x, 2, 4, rows);
        for (int r = 0; r < 4; r++) {
            float one = janas_mxfp4_dot_ref(w + 16 * r, x, 2);
            CHECK(rows[r] == one || (isnan(rows[r]) && isnan(one)),
                  "mxfp4 rows %d: %g vs %g", r, rows[r], one);
        }
        if (trial % 4 == 3)
            continue;
        /* the exact rewrite into Q8_0: the same products, bit for bit */
        size_t tiny = 0;
        CHECK(janas_mxfp4_to_q8_0((const uint8_t *)w, w8, NB * 8, &tiny) == 0 &&
                  tiny == 0,
              "mxfp4 to q8_0 (%zu tiny)", tiny);
        float q8 = janas_dot(JANAS_Q8_0, w8, x, NB);
        CHECK(q8 == fast, "mxfp4 %g vs its q8_0 rewrite %g", fast, q8);
    }
    free(w);
    free(w8);
    free(x);
    free(xf);
    free(wf);
}

static void test_dot_q4_0_q5_1(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    struct janas_block_q4_0 *w4 = malloc(NB * 8 * sizeof(*w4));
    struct janas_block_q5_1 *w5 = malloc(NB * 8 * sizeof(*w5));
    struct janas_block_q4_1 *w1 = malloc(NB * 8 * sizeof(*w1));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float));
    for (int trial = 0; trial < 100; trial++) {
        unsigned char *p4 = (unsigned char *)w4, *p5 = (unsigned char *)w5;
        for (size_t i = 0; i < NB * 8 * sizeof(*w4); i++)
            p4[i] = (unsigned char)rnd();
        for (size_t i = 0; i < NB * 8 * sizeof(*w5); i++)
            p5[i] = (unsigned char)rnd();
        unsigned char *p1 = (unsigned char *)w1;
        for (size_t i = 0; i < NB * 8 * sizeof(*w1); i++)
            p1[i] = (unsigned char)rnd();
        for (int b = 0; b < NB * 8; b++) {
            w4[b].d = janas_fp32_to_fp16(rndf(-0.01f, 0.01f));
            w5[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
            w5[b].m = janas_fp32_to_fp16(rndf(-0.1f, 0.0f));
            w1[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
            w1[b].m = janas_fp32_to_fp16(rndf(-0.1f, 0.0f));
        }
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }
        for (int five = 0; five < 3; five++) {
            const void *w = five == 2 ? (const void *)w1
                            : five    ? (const void *)w5
                                      : (const void *)w4;
            int t = five == 2 ? JANAS_Q4_1 : five ? JANAS_Q5_1 : JANAS_Q4_0;
            const char *name = five == 2 ? "q4_1" : five ? "q5_1" : "q4_0";
            if (five == 2)
                janas_q4_1_dequantize(w1, wf, N);
            else if (five)
                janas_q5_1_dequantize(w5, wf, N);
            else
                janas_q4_0_dequantize(w4, wf, N);
            double expect = 0.0, mag = 0.0;
            for (int b = 0; b < NB; b++)
                for (int i = 0; i < JANAS_QK; i++) {
                    double e = (double)wf[b * JANAS_QK + i] * act(x, b, i, t);
                    expect += e;
                    mag += fabs(e);
                }
            float ref = five == 2 ? janas_q4_1_dot_ref(w1, x, NB)
                        : five    ? janas_q5_1_dot_ref(w5, x, NB)
                                  : janas_q4_0_dot_ref(w4, x, NB);
            CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "%s ref %g vs %g",
                  name, ref, expect);
            float fast = janas_dot(t, w, x, NB);
            CHECK(fast == ref, "%s fast %g vs ref %g", name, fast, ref);
            for (int nv = 1; nv <= NV; nv++) {
                float out[NV];
                janas_dot_multi(t, w, x, NB, (size_t)nv, NB, out, 1);
                for (int v = 0; v < nv; v++) {
                    float r = five == 2 ? janas_q4_1_dot_ref(w1, x + v * NB, NB)
                              : five    ? janas_q5_1_dot_ref(w5, x + v * NB, NB)
                                     : janas_q4_0_dot_ref(w4, x + v * NB, NB);
                    CHECK(out[v] == r, "%s multi nv %d v %d: %g vs %g", name,
                          nv, v, out[v], r);
                }
            }
        }
    }
    /* by hand: Q5_1 weight 17 takes bit 17 of qh and the high nibble of
       byte 1; weight 1 is only the minimum */
    struct janas_block_q5_1 h = {0};
    h.d = janas_fp32_to_fp16(1.0f);
    h.m = janas_fp32_to_fp16(-2.0f);
    h.qh[2] = 0x02; /* bit 17 */
    h.qs[1] = 0x30; /* weight 17: low bits 3 */
    float y[32];
    janas_q5_1_dequantize(&h, y, 32);
    CHECK(y[17] == 19.0f - 2.0f && y[1] == -2.0f, "q5_1 by hand: %g %g", y[17],
          y[1]);
    free(w4);
    free(w5);
    free(w1);
    free(x);
    free(xf);
    free(wf);
}

/* Random bytes for a row of n weights of a type in blocks of 32 or 64, each
   block's scale a finite fp16 (sizes from janas_qtype_row_bytes). */
static void random_blocks(int type, unsigned char *w, size_t n)
{
    size_t bytes = janas_qtype_row_bytes(type, n);
    for (size_t i = 0; i < bytes; i++)
        w[i] = (unsigned char)rnd();
    size_t size = type == JANAS_Q2_0   ? sizeof(struct janas_block_q2_0)
                  : type == JANAS_Q5_0 ? sizeof(struct janas_block_q5_0)
                  : type == JANAS_Q8_0 ? sizeof(struct janas_block_q8_0)
                                       : sizeof(struct janas_block_iq4nl);
    for (size_t b = 0; b < bytes / size; b++) {
        uint16_t d = janas_fp32_to_fp16(rndf(-0.01f, 0.01f));
        memcpy(w + b * size, &d, 2);
    }
}

/*
 * janas_dot_multi_rows on Q2_0 and the K types, on long rows: the vectors
 * reordered (Q2_0) or their scales and sums made ready (Q4_K, Q5_K, Q6_K) a few
 * at a time, or not at all where the rows are too long (130 groups), 1-65
 * vectors; each result equal to janas_dot's to the bit, and Q2_0's to the
 * reference.
 */
static void test_multi_rows_long(void)
{
    /* 11 rows: Q4_K's tiles take eight (q4k_gemm), the products the
       rest; up to 65 vectors, then 300: past the tiles' 256 tokens at a
       time */
    enum { ROWS = 11, NV = 300 };
    static const size_t lens[] = {2, 48, 130};
    static const int types[] = {
        JANAS_Q2_0,    JANAS_Q4_K,    JANAS_Q5_K,   JANAS_Q6_K,  JANAS_Q8_0,
        JANAS_Q4_0,    JANAS_Q5_0,    JANAS_IQ4_NL, JANAS_MXFP4, JANAS_Q4_1,
        JANAS_Q5_1,    JANAS_Q3_K,    JANAS_IQ4_XS, JANAS_Q2_K,  JANAS_IQ3_S,
        JANAS_IQ3_XXS, JANAS_IQ2_XXS, JANAS_IQ2_XS, JANAS_IQ2_S, JANAS_IQ1_S,
        JANAS_IQ1_M};
    for (unsigned ti = 0; ti < sizeof(types) / sizeof(*types); ti++)
        for (unsigned li = 0; li < sizeof(lens) / sizeof(*lens); li++) {
            int t = types[ti];
            size_t nb = lens[li], n = nb * JANAS_QK;
            size_t rb = janas_qtype_row_bytes(t, n);
            unsigned char *w = malloc(ROWS * rb);
            struct janas_block_q8k *x = malloc(NV * nb * sizeof(*x));
            float *xf = malloc(n * sizeof(float)), out[ROWS * NV];
            for (int r = 0; r < ROWS; r++) {
                if (t == JANAS_Q2_0 || t == JANAS_Q8_0 || t == JANAS_Q4_0 ||
                    t == JANAS_Q5_0 || t == JANAS_IQ4_NL)
                    random_blocks(t, w + r * rb, n);
                else if (t == JANAS_MXFP4 || t == JANAS_Q4_1 ||
                         t == JANAS_Q5_1) {
                    /* blocks of 32: MXFP4's power of two, Q4_1's and
                       Q5_1's d and m, finite */
                    size_t bs = janas_qtype_row_bytes(t, 32);
                    unsigned char *q = w + r * rb;
                    for (size_t i = 0; i < rb; i++)
                        q[i] = (unsigned char)rnd();
                    for (size_t i = 0; i < rb; i += bs) {
                        if (t == JANAS_MXFP4) {
                            q[i] = (unsigned char)(118 + rnd() % 12);
                            continue;
                        }
                        uint16_t d[2] = {
                            janas_fp32_to_fp16(rndf(-0.01f, 0.01f)),
                            janas_fp32_to_fp16(rndf(-0.01f, 0.01f))};
                        memcpy(q + i, d, 4);
                    }
                } else
                    for (size_t b = 0; b < nb; b++) {
                        /* Q4_K and Q5_K: d and dmin first, the IQ types d;
                           Q6_K and Q3_K: d last, Q2_K d and dmin; IQ1_M: d
                           in the top nibbles of its four scale words */
                        size_t bs = janas_qtype_block_size(t);
                        unsigned char *q = w + r * rb + b * bs;
                        for (size_t i = 0; i < bs; i++)
                            q[i] = (unsigned char)rnd();
                        uint16_t d[2] = {
                            janas_fp32_to_fp16(rndf(-0.01f, 0.01f)),
                            janas_fp32_to_fp16(rndf(-0.01f, 0.01f))};
                        if (t == JANAS_Q6_K || t == JANAS_Q3_K)
                            memcpy(q + bs - 2, d, 2);
                        else if (t == JANAS_Q2_K)
                            memcpy(q + bs - 4, d, 4);
                        else if (t == JANAS_IQ1_M) {
                            uint16_t sc[4];
                            memcpy(sc, q + bs - 8, 8);
                            for (int k = 0; k < 4; k++)
                                sc[k] = (uint16_t)((sc[k] & 0x0fff) |
                                                   (d[0] >> 4 * k & 15) << 12);
                            memcpy(q + bs - 8, sc, 8);
                        } else
                            memcpy(q, d, 4);
                    }
            }
            for (int v = 0; v < NV; v++) {
                for (size_t i = 0; i < n; i++)
                    xf[i] = rndf(-1.0f, 1.0f);
                janas_q8k_quantize(xf, x + v * nb, n);
            }
            for (int nv = 1; nv <= NV; nv = nv < 9     ? nv + 1
                                            : nv < 65  ? nv + 8
                                            : nv == 65 ? NV
                                                       : NV + 1) {
                janas_dot_multi_rows(t, w, rb, x, nb, (size_t)nv, nb, ROWS, out,
                                     ROWS);
                for (int v = 0; v < nv; v++)
                    for (int r = 0; r < ROWS; r++) {
                        float one = janas_dot(t, w + r * rb, x + v * nb, nb);
                        CHECK(out[r + ROWS * v] == one,
                              "type %d %zu groups nv %d r %d v %d: %g vs %g", t,
                              nb, nv, r, v, out[r + ROWS * v], one);
                        if (t == JANAS_Q2_0) {
                            float ref = janas_q2_0_dot_ref(
                                (const void *)(w + r * rb), x + v * nb, nb);
                            CHECK(one == ref, "q2_0 %zu groups: %g vs %g", nb,
                                  one, ref);
                        }
                    }
            }
            float one[ROWS];
            janas_dot_rows(t, w, rb, x, nb, ROWS, one);
            for (int r = 0; r < ROWS; r++)
                CHECK(one[r] == janas_dot(t, w + r * rb, x, nb),
                      "type %d %zu groups rows %d", t, nb, r);
            free(w);
            free(x);
            free(xf);
        }
}

/*
 * Q5_0 and Q2_0: the reference against a float product over the
 * dequantized weights, the kernels (single, 1-17 vectors, rows) equal to it
 * to the bit; the values by hand as ggml's dequantize_row_q5_0 and
 * dequantize_row_q2_0 give them.
 */
static void test_dot_q5_0_q2_0(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    static const int types[] = {JANAS_Q5_0, JANAS_Q2_0};
    unsigned char *w = malloc(NB * janas_qtype_block_size(JANAS_Q5_0));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float));
    float rows[4];
    for (int trial = 0; trial < 100; trial++)
        for (unsigned k = 0; k < sizeof(types) / sizeof(*types); k++) {
            int t = types[k];
            const char *name = t == JANAS_Q5_0 ? "q5_0" : "q2_0";
            random_blocks(t, w, N);
            for (int v = 0; v < NV; v++) {
                for (int i = 0; i < N; i++)
                    xf[i] = rndf(-1.0f, 1.0f);
                janas_q8k_quantize(xf, x + v * NB, N);
            }
            janas_dequantize(t, w, wf, N);
            double expect = 0.0, mag = 0.0;
            for (int b = 0; b < NB; b++)
                for (int i = 0; i < JANAS_QK; i++) {
                    double e = (double)wf[b * JANAS_QK + i] * act(x, b, i, t);
                    expect += e;
                    mag += fabs(e);
                }
            float ref = t == JANAS_Q5_0
                            ? janas_q5_0_dot_ref((const void *)w, x, NB)
                            : janas_q2_0_dot_ref((const void *)w, x, NB);
            CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "%s ref %g vs %g",
                  name, ref, expect);
            float fast = janas_dot(t, w, x, NB);
            CHECK(fast == ref, "%s fast %g vs ref %g", name, fast, ref);
            for (int nv = 1; nv <= NV; nv++) {
                float out[NV];
                janas_dot_multi(t, w, x, NB, (size_t)nv, NB, out, 1);
                for (int v = 0; v < nv; v++) {
                    float r = t == JANAS_Q5_0
                                  ? janas_q5_0_dot_ref((const void *)w,
                                                       x + v * NB, NB)
                                  : janas_q2_0_dot_ref((const void *)w,
                                                       x + v * NB, NB);
                    CHECK(out[v] == r, "%s multi nv %d v %d: %g vs %g", name,
                          nv, v, out[v], r);
                }
            }
            /* four rows of two groups each, the matrix read row by row */
            size_t gb = janas_qtype_block_size(t);
            janas_dot_rows(t, w, 2 * gb, x, 2, 4, rows);
            for (int r = 0; r < 4; r++) {
                float one = janas_dot(t, w + 2 * gb * (size_t)r, x, 2);
                CHECK(rows[r] == one, "%s rows %d: %g vs %g", name, r, rows[r],
                      one);
            }
            /* the same four rows on 1-17 vectors at once */
            for (int nv = 1; nv <= NV; nv++) {
                float out[4 * NV];
                janas_dot_multi_rows(t, w, 2 * gb, x, NB, (size_t)nv, 2, 4, out,
                                     4);
                for (int r = 0; r < 4; r++)
                    for (int v = 0; v < nv; v++) {
                        float one =
                            janas_dot(t, w + 2 * gb * (size_t)r, x + v * NB, 2);
                        CHECK(out[r + 4 * v] == one,
                              "%s multi rows nv %d r %d v %d: %g vs %g", name,
                              nv, r, v, out[r + 4 * v], one);
                    }
            }
        }
    /* by hand. Q5_0: weight 17 takes bit 17 of qh and the high nibble of
       byte 1, weight 1 the low nibble alone, less 16 */
    struct janas_block_q5_0 h5 = {0};
    h5.d = janas_fp32_to_fp16(1.0f);
    h5.qh[2] = 0x02; /* bit 17 */
    h5.qs[1] = 0x37; /* weight 1: 7; weight 17: 3 */
    float y[64];
    janas_q5_0_dequantize(&h5, y, 32);
    CHECK(y[17] == 19.0f - 16.0f && y[1] == 7.0f - 16.0f && y[0] == -16.0f,
          "q5_0 by hand: %g %g %g", y[17], y[1], y[0]);
    /* Q2_0: weight i in bits 2 (i % 4) of byte i / 4, 00 = -1, 11 = +2 */
    struct janas_block_q2_0 h2 = {0};
    h2.d = janas_fp32_to_fp16(0.5f);
    h2.qs[0] = 0xe4;  /* weights 0-3: 0, 1, 2, 3 */
    h2.qs[15] = 0x1b; /* weights 60-63: 3, 2, 1, 0 */
    janas_q2_0_dequantize(&h2, y, 64);
    CHECK(y[0] == -0.5f && y[1] == 0.0f && y[2] == 0.5f && y[3] == 1.0f &&
              y[60] == 1.0f && y[61] == 0.5f && y[62] == 0.0f &&
              y[63] == -0.5f && y[4] == -0.5f,
          "q2_0 by hand: %g %g %g %g / %g %g %g %g", y[0], y[1], y[2], y[3],
          y[60], y[61], y[62], y[63]);
    free(w);
    free(x);
    free(xf);
    free(wf);
}

/*
 * Rows that end inside a group of 256 (Qwen3.8-Flash-Next's experts, 640
 * wide), in the types stored in blocks of 32 or 64: the product is the whole
 * groups' plus the tail padded with zero blocks, the same bits whichever way
 * it is asked for (one row, rows, 1-17 vectors, the grouped products on a
 * pool); and it is the float product over the n weights.
 */
static void test_row_tails(void)
{
    /* 33 vectors and 37 rows: the tiles take 32 of them (from 24 vectors),
       the tails 16 rows at a time */
    enum { NMAX = 768, NBX = NMAX / JANAS_QK, NV = 33, ROWS = 37 };
    static const int types[] = {JANAS_Q2_0, JANAS_Q5_0, JANAS_IQ4_NL,
                                JANAS_Q4_0, JANAS_Q8_0};
    static const size_t ns[] = {640, 64, 320, 576, 704};
    size_t big = ROWS * janas_qtype_row_bytes(JANAS_Q8_0, NMAX);
    unsigned char *w = malloc(big);
    struct janas_block_q8k *x = malloc(NV * NBX * sizeof(*x));
    float xf[NMAX], *wf = malloc(NMAX * sizeof(float));
    float y[NV * ROWS];
    struct janas_pool *pool = janas_pool_create(3, NULL);
    CHECK(pool != NULL, "pool");
    for (unsigned k = 0; k < sizeof(types) / sizeof(*types); k++)
        for (unsigned j = 0; j < sizeof(ns) / sizeof(*ns); j++) {
            int t = types[k];
            size_t n = ns[j], nb = n / JANAS_QK, nbx = (n + 255) / 256;
            size_t rb = janas_qtype_row_bytes(t, n);
            if (t == JANAS_Q2_0 && n % 64) {
                CHECK(rb == 0, "q2_0 row of %zu: %zu bytes", n, rb);
                continue;
            }
            CHECK(rb != 0 && rb < (nb + 1) * janas_qtype_block_size(t),
                  "type %d row of %zu: %zu bytes", t, n, rb);
            for (int r = 0; r < ROWS; r++)
                random_blocks(t, w + (size_t)r * rb, n);
            /* activations past n zero, as the engine pads them */
            for (int v = 0; v < NV; v++) {
                for (size_t i = 0; i < NMAX; i++)
                    xf[i] = i < n ? rndf(-1.0f, 1.0f) : 0.0f;
                janas_q8k_quantize(xf, x + v * nbx, nbx * JANAS_QK);
            }
            /* the float product of row 0 */
            janas_dequantize(t, w, wf, n);
            double expect = 0.0, mag = 0.0;
            for (size_t i = 0; i < n; i++) {
                double e =
                    (double)wf[i] * act(x, (int)(i / 256), (int)(i % 256), t);
                expect += e;
                mag += fabs(e);
            }
            float one = janas_dot_n(t, w, x, n);
            CHECK(fabs(one - expect) <= 1e-5 * mag + 1e-6,
                  "type %d n %zu: %g vs %g", t, n, one, expect);
            /* its definition: whole groups, then the padded tail */
            _Alignas(64) unsigned char pad[8 * sizeof(struct janas_block_q8_0)];
            size_t full = nb * janas_qtype_block_size(t);
            memset(pad, 0, sizeof(pad));
            memcpy(pad, w + full, rb - full);
            float def = (nb ? janas_dot(t, w, x, nb) : 0.0f) +
                        janas_dot(t, pad, x + nb, 1);
            CHECK(one == def, "type %d n %zu: %g vs its definition %g", t, n,
                  one, def);
            janas_dot_rows_n(t, w, rb, x, n, ROWS, y);
            for (int r = 0; r < ROWS; r++) {
                float e = janas_dot_n(t, w + (size_t)r * rb, x, n);
                CHECK(y[r] == e, "type %d n %zu rows %d: %g vs %g", t, n, r,
                      y[r], e);
            }
            for (int nv = 1; nv <= NV; nv += 4) {
                float out[NV];
                janas_dot_multi_n(t, w, x, nbx, (size_t)nv, n, out, 1);
                for (int v = 0; v < nv; v++) {
                    float e = janas_dot_n(t, w, x + v * nbx, n);
                    CHECK(out[v] == e, "type %d n %zu multi %d/%d: %g vs %g", t,
                          n, v, nv, out[v], e);
                }
            }
            for (int nv = 1; nv <= NV; nv += 8) {
                janas_dot_multi_rows_n(t, w, rb, x, nbx, (size_t)nv, n, ROWS, y,
                                       ROWS);
                for (int v = 0; v < nv; v++)
                    for (int r = 0; r < ROWS; r++) {
                        float e =
                            janas_dot_n(t, w + (size_t)r * rb, x + v * nbx, n);
                        CHECK(y[v * ROWS + r] == e,
                              "type %d n %zu multi rows nv %d v %d row %d: %g "
                              "vs %g",
                              t, n, nv, v, r, y[v * ROWS + r], e);
                    }
            }
            /* the grouped products: one vector, five and NV */
            for (int nv = 1; pool && nv <= NV; nv = nv == 1   ? 5
                                                    : nv == 5 ? NV
                                                              : NV + 1) {
                struct janas_matvec_task task = {.type = t,
                                                 .w = w,
                                                 .x = x,
                                                 .y = y,
                                                 .rows = ROWS,
                                                 .cols = n,
                                                 .n_vec = (size_t)nv,
                                                 .x_stride = nbx,
                                                 .y_stride = ROWS};
                janas_matvec_q4k_group(pool, &task, 1);
                for (int v = 0; v < nv; v++)
                    for (int r = 0; r < ROWS; r++) {
                        float e =
                            janas_dot_n(t, w + (size_t)r * rb, x + v * nbx, n);
                        CHECK(y[v * ROWS + r] == e,
                              "type %d n %zu group nv %d v %d row %d: %g vs "
                              "%g",
                              t, n, nv, v, r, y[v * ROWS + r], e);
                    }
            }
        }
    janas_pool_destroy(pool);
    free(w);
    free(x);
    free(wf);
}

/*
 * Unpacking a whole row: janas_dequantize has to reach the right unpacker for
 * every type the engine accepts in a model file, not just the two it used to
 * know. A token embedding is read this way, and reading a Q8_0 one as Q4_K
 * gave numbers so large that the first layer produced NaN - the model loaded,
 * passed its checksums and answered nonsense (reported from an i7-1365U, on a
 * GGUF whose token_embd.weight is Q8_0).
 */
/* Q8_0 rows on float vectors (bf16.h): one row, a row on four vectors and
   four rows on one vector, each the bits of the order bf16.h states, scales
   subnormal and zero among them */
static float q8w_ref(const unsigned char *w, const float *x, uint32_t n)
{
    float acc[8] = {0};
    for (uint32_t b = 0; b < n / 32; b++, w += 34, x += 32) {
        const int8_t *q = (const int8_t *)(w + 2);
        uint16_t h;
        memcpy(&h, w, 2);
        float d = janas_fp16_to_fp32(h), bs[8] = {0};
        for (int g = 0; g < 4; g++)
            for (int k = 0; k < 8; k++)
                bs[k] = fmaf((float)q[8 * g + k], x[8 * g + k], bs[k]);
        for (int k = 0; k < 8; k++)
            acc[k] = fmaf(bs[k], d, acc[k]);
    }
    return ((acc[0] + acc[4]) + (acc[2] + acc[6])) +
           ((acc[1] + acc[5]) + (acc[3] + acc[7]));
}

static void test_q8w(void)
{
    enum { N = 320, ROW = N / 32 * 34 };
    unsigned char *w = malloc(4 * ROW);
    float *x = malloc(4 * N * sizeof(float));
    for (int rep = 0; rep < 20; rep++) {
        for (int i = 0; i < 4 * ROW; i++)
            w[i] = (unsigned char)rnd();
        for (int b = 0; b < 4 * N / 32; b++) {
            float d = rndf(0.0001f, 0.05f);
            uint16_t h = janas_fp32_to_fp16(d);
            if (b % 7 == 3)
                h = (uint16_t)(rnd() & 0x83ff); /* subnormal or zero */
            memcpy(w + b * 34, &h, 2);
        }
        for (int i = 0; i < 4 * N; i++)
            x[i] = rndf(-2.0f, 2.0f);
        float o[4], e;
        const float *xs[4] = {x, x + N, x + 2 * N, x + 3 * N};
        janas_q8w_dot4(w, xs, N, o);
        for (int t = 0; t < 4; t++) {
            e = q8w_ref(w, xs[t], N);
            CHECK(!memcmp(&o[t], &e, 4), "q8w dot4 vector %d: %g != %g", t,
                  o[t], e);
        }
        janas_q8w_dot4r(w, ROW, x, N, o);
        for (int r = 0; r < 4; r++) {
            e = q8w_ref(w + r * ROW, x, N);
            CHECK(!memcmp(&o[r], &e, 4), "q8w dot4r row %d: %g != %g", r, o[r],
                  e);
            float one = janas_q8w_dot(w + r * ROW, x, N);
            CHECK(!memcmp(&one, &e, 4), "q8w dot row %d: %g != %g", r, one, e);
        }
    }
    free(w);
    free(x);
}

/* bf16 rows on one vector four at a time (bf16.h) against one at a time,
   lengths with and without a tail; rounding to bf16 against the scalar
   rule, ties, infinities and NaNs among the values */
static void test_bf16_rows(void)
{
    enum { N = 333, ROW = 340 };
    uint16_t *w = malloc(4 * ROW * sizeof(uint16_t));
    float *x = malloc(ROW * sizeof(float));
    for (int rep = 0; rep < 20; rep++) {
        for (int i = 0; i < 4 * ROW; i++)
            w[i] = (uint16_t)(janas_f2u(rndf(-0.1f, 0.1f)) >> 16);
        for (int i = 0; i < ROW; i++)
            x[i] = rndf(-2.0f, 2.0f);
        uint32_t n = rep % 2 ? N : 320;
        float o[4];
        janas_bf16_dot4r(w, ROW, x, n, o);
        for (int r = 0; r < 4; r++) {
            float e = janas_bf16_dot(w + r * ROW, x, n);
            CHECK(!memcmp(&o[r], &e, 4), "bf16 dot4r row %d n %u: %g != %g", r,
                  n, o[r], e);
        }
    }
    enum { M = 4099 };
    float *a = malloc(M * sizeof(float)), *b = malloc(M * sizeof(float));
    for (int i = 0; i < M; i++) {
        uint32_t u = rnd();
        if (i % 5 == 1)
            u = (u & 0xffff0000u) | 0x8000u; /* a tie */
        if (i % 97 == 3)
            u = 0x7f800000u | (u & 0x80000000u); /* infinity */
        if (i % 89 == 5)
            u = 0x7f800001u | (u & 0x807fffffu); /* NaN */
        a[i] = janas_u2f(u);
    }
    janas_round_bf16_n(a, b, M);
    for (int i = 0; i < M; i++) {
        uint32_t u = janas_f2u(a[i]), e = u;
        if ((u & 0x7fffffffu) <= 0x7f800000u)
            e = (u + (0x7fffu + ((u >> 16) & 1u))) & 0xffff0000u;
        CHECK(janas_f2u(b[i]) == e, "round bf16 %08x: %08x != %08x", u,
              janas_f2u(b[i]), e);
    }
    free(w);
    free(x);
    free(a);
    free(b);
}

static void test_dequantize(void)
{
    enum { NB = 8, N = NB * JANAS_QK };
    static const int types[] = {JANAS_Q4_K,  JANAS_Q5_K,   JANAS_Q6_K,
                                JANAS_Q8_0,  JANAS_IQ4_NL, JANAS_IQ4_XS,
                                JANAS_Q4_0,  JANAS_Q5_1,   JANAS_Q4_1,
                                JANAS_MXFP4, JANAS_Q5_0,   JANAS_Q2_0};
    float *a = malloc(N * sizeof(float)), *b = malloc(N * sizeof(float));
    for (unsigned k = 0; k < sizeof(types) / sizeof(*types); k++) {
        int type = types[k];
        size_t bytes = NB * janas_qtype_block_size(type);
        CHECK(bytes != 0, "type %d: no block size", type);
        unsigned char *w = malloc(bytes);
        for (size_t i = 0; i < bytes; i++)
            w[i] = (unsigned char)rnd();
        /* finite scales, so the two ways can be compared bit for bit */
        for (int i = 0; i < NB; i++)
            switch (type) {
            case JANAS_Q4_K: {
                struct janas_block_q4k *q = (struct janas_block_q4k *)w + i;
                q->d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                q->dmin = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                break;
            }
            case JANAS_Q5_K: {
                struct janas_block_q5k *q = (struct janas_block_q5k *)w + i;
                q->d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                q->dmin = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                break;
            }
            case JANAS_Q6_K: {
                struct janas_block_q6k *q = (struct janas_block_q6k *)w + i;
                q->d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                break;
            }
            case JANAS_IQ4_NL: { /* eight blocks cover one super-block */
                struct janas_block_iq4nl *q =
                    (struct janas_block_iq4nl *)w + i * 8;
                for (int j = 0; j < 8; j++)
                    q[j].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                break;
            }
            case JANAS_IQ4_XS: {
                struct janas_block_iq4xs *q = (struct janas_block_iq4xs *)w + i;
                q->d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                break;
            }
            case JANAS_MXFP4: { /* eight blocks cover one super-block */
                struct janas_block_mxfp4 *q =
                    (struct janas_block_mxfp4 *)w + i * 8;
                for (int j = 0; j < 8; j++)
                    q[j].e = (uint8_t)(118 + rnd() % 14);
                break;
            }
            case JANAS_Q4_0: {
                struct janas_block_q4_0 *q =
                    (struct janas_block_q4_0 *)w + i * 8;
                for (int j = 0; j < 8; j++)
                    q[j].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                break;
            }
            case JANAS_Q4_1: {
                struct janas_block_q4_1 *q =
                    (struct janas_block_q4_1 *)w + i * 8;
                for (int j = 0; j < 8; j++) {
                    q[j].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                    q[j].m = janas_fp32_to_fp16(rndf(-0.1f, 0.0f));
                }
                break;
            }
            case JANAS_Q5_1: {
                struct janas_block_q5_1 *q =
                    (struct janas_block_q5_1 *)w + i * 8;
                for (int j = 0; j < 8; j++) {
                    q[j].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                    q[j].m = janas_fp32_to_fp16(rndf(-0.1f, 0.0f));
                }
                break;
            }
            case JANAS_Q5_0: {
                struct janas_block_q5_0 *q =
                    (struct janas_block_q5_0 *)w + i * 8;
                for (int j = 0; j < 8; j++)
                    q[j].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                break;
            }
            case JANAS_Q2_0: { /* four blocks cover one super-block */
                struct janas_block_q2_0 *q =
                    (struct janas_block_q2_0 *)w + i * 4;
                for (int j = 0; j < 4; j++)
                    q[j].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
                break;
            }
            default: { /* eight Q8_0 blocks cover one super-block */
                struct janas_block_q8_0 *q =
                    (struct janas_block_q8_0 *)w + i * 8;
                for (int j = 0; j < 8; j++)
                    q[j].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
            }
            }
        switch (type) {
        case JANAS_Q4_K:
            janas_q4k_dequantize((const void *)w, a, N);
            break;
        case JANAS_Q5_K:
            janas_q5k_dequantize((const void *)w, a, N);
            break;
        case JANAS_Q6_K:
            janas_q6k_dequantize((const void *)w, a, N);
            break;
        case JANAS_IQ4_NL:
            janas_iq4nl_dequantize((const void *)w, a, N);
            break;
        case JANAS_IQ4_XS:
            janas_iq4xs_dequantize((const void *)w, a, N);
            break;
        case JANAS_MXFP4:
            janas_mxfp4_dequantize((const void *)w, a, N);
            break;
        case JANAS_Q4_0:
            janas_q4_0_dequantize((const void *)w, a, N);
            break;
        case JANAS_Q5_1:
            janas_q5_1_dequantize((const void *)w, a, N);
            break;
        case JANAS_Q4_1:
            janas_q4_1_dequantize((const void *)w, a, N);
            break;
        case JANAS_Q5_0:
            janas_q5_0_dequantize((const void *)w, a, N);
            break;
        case JANAS_Q2_0:
            janas_q2_0_dequantize((const void *)w, a, N);
            break;
        default:
            janas_q8_0_dequantize((const void *)w, a, N);
        }
        janas_dequantize(type, (const void *)w, b, N);
        CHECK(memcmp(a, b, N * sizeof(float)) == 0,
              "type %d: janas_dequantize differs from its own unpacker", type);
        /* the stride a row of a table is read with, as embed() computes it,
           has to be the one the unpacker walks: the second half of this
           buffer read as a row of its own must give the same numbers */
        size_t half = N / 2,
               row_bytes = half / JANAS_QK * janas_qtype_block_size(type);
        janas_dequantize(type, (const void *)(w + row_bytes), b, half);
        CHECK(memcmp(a + half, b, half * sizeof(float)) == 0,
              "type %d: a row read at its own offset differs", type);
        free(w);
    }
    free(a);
    free(b);
}

/*
 * The vector quantization of an activation must be what the scalar loop
 * gives, to the bit: the engine's numbers stay reproducible whatever the
 * machine, and a difference here would move every logit that follows.
 */
static void q8k_ref(const float *x, struct janas_block_q8k *y, size_t n,
                    int det)
{
    for (size_t b = 0; b < n / JANAS_QK; b++, x += JANAS_QK) {
        float amax = 0.0f;
        for (int i = 0; i < JANAS_QK; i++)
            amax = fmaxf(amax, fabsf(x[i]));
        float d = det ? amax * 0.007874015718698502f : amax / 127.0f;
        float id = d > 0.0f ? (det ? janas_recip_det(d) : 1.0f / d) : 0.0f;
        y[b].d = d;
        for (int i = 0; i < JANAS_QK; i++) {
            float q = nearbyintf(x[i] * id);
            y[b].qs[i] = (int8_t)(q > 127.0f ? 127 : (q < -127.0f ? -127 : q));
        }
        for (int g = 0; g < JANAS_QK / 16; g++) {
            int s = 0;
            for (int i = 0; i < 16; i++)
                s += y[b].qs[g * 16 + i];
            y[b].bsums[g] = (int16_t)s;
        }
        for (int k = 0; k < JANAS_QK / 32; k++)
            y[b].bsums2[k] =
                (int16_t)(y[b].bsums[2 * k] + y[b].bsums[2 * k + 1]);
        for (int k = 0; k < 8; k++) {
            float m = 0.0f;
            for (int i = 0; i < 32; i++)
                m = fmaxf(m, fabsf(x[32 * k + i]));
            float dk = det ? m * 0.007874015718698502f : m / 127.0f;
            float ik =
                dk > 0.0f ? (det ? janas_recip_det(dk) : 1.0f / dk) : 0.0f;
            y[b].d32[k] = dk;
            for (int i = 0; i < 32; i++) {
                float q = nearbyintf(x[32 * k + i] * ik);
                y[b].qs32[32 * k + i] =
                    (int8_t)(q > 127.0f ? 127 : (q < -127.0f ? -127 : q));
            }
        }
        for (int g = 0; g < JANAS_QK / 16; g++) {
            int s = 0;
            for (int i = 0; i < 16; i++)
                s += y[b].qs32[g * 16 + i];
            y[b].bsums32[g] = (int16_t)s;
        }
    }
}

static void test_quantize_q8k(void)
{
    enum { NB = 6, N = NB * JANAS_QK };
    static float x[N];
    static struct janas_block_q8k got[NB], want[NB];
    for (int round = 0; round < 8; round++) {
        for (int i = 0; i < N; i++)
            switch (round) {
            case 0:
                x[i] = 0.0f;
                break; /* every weight zero */
            case 1:
                x[i] = rndf(-1.0f, 1.0f);
                break;
            case 2:
                x[i] = rndf(-1e-6f, 1e-6f);
                break; /* tiny */
            case 3:
                x[i] = rndf(-1e6f, 1e6f);
                break; /* large */
            case 4:
                x[i] = (i % 2) ? -0.0f : 0.0f;
                break;
            /* values that land exactly halfway, where rounding must tie to
               even the same way in both */
            case 5:
                x[i] = (float)((i % 255) - 127) + 0.5f;
                break;
            case 6:
                x[i] = (float)((i % 255) - 127);
                break;
            default:
                x[i] = rndf(-3.0f, 3.0f) * (i % 7 ? 1.0f : 1e4f);
                break;
            }
        for (int det = 0; det < 2; det++) {
            if (det)
                janas_q8k_quantize_det(x, got, N);
            else
                janas_q8k_quantize(x, got, N);
            q8k_ref(x, want, N, det);
            CHECK(memcmp(got, want, sizeof(want)) == 0,
                  "q8k quantize%s round %d differs from the reference",
                  det ? " (det)" : "", round);
        }
    }
}

/* Q3_K: the reference against a float product over the dequantized
   weights, the kernels on 1-17 vectors equal to it to the bit, and
   janas_dequantize reaching its unpacker. */
static void test_dot_q3k(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    struct janas_block_q3k *w = malloc(NB * sizeof(*w));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float)),
          *w2 = malloc(N * sizeof(float));
    for (int trial = 0; trial < 100; trial++) {
        unsigned char *p = (unsigned char *)w;
        for (size_t i = 0; i < NB * sizeof(*w); i++)
            p[i] = (unsigned char)rnd();
        for (int b = 0; b < NB; b++)
            w[b].d = janas_fp32_to_fp16(rndf(-0.01f, 0.01f));
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }
        janas_q3k_dequantize(w, wf, N);
        janas_dequantize(JANAS_Q3_K, w, w2, N);
        CHECK(memcmp(wf, w2, N * sizeof(float)) == 0,
              "q3k: janas_dequantize differs from its unpacker");
        double expect = 0.0, mag = 0.0;
        for (int b = 0; b < NB; b++)
            for (int i = 0; i < JANAS_QK; i++) {
                double e = (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];
                expect += e;
                mag += fabs(e);
            }
        float ref = janas_q3k_dot_ref(w, x, NB);
        CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "q3k ref %g vs %g", ref,
              expect);
        CHECK(janas_dot(JANAS_Q3_K, w, x, NB) == ref, "q3k fast vs ref");
        for (int nv = 1; nv <= NV; nv++) {
            float out[NV];
            janas_dot_multi(JANAS_Q3_K, w, x, NB, (size_t)nv, NB, out, 1);
            for (int v = 0; v < nv; v++)
                CHECK(out[v] == janas_q3k_dot_ref(w, x + v * NB, NB),
                      "q3k multi nv %d v %d", nv, v);
        }
    }
    free(w);
    free(x);
    free(xf);
    free(wf);
    free(w2);
}

/* IQ3_S: the reference against a float product over the dequantized
   weights, the kernels on 1-17 vectors equal to it to the bit, and
   janas_dequantize reaching its unpacker. */
static void test_dot_iq3s(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    struct janas_block_iq3s *w = malloc(NB * sizeof(*w));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float)),
          *w2 = malloc(N * sizeof(float));
    for (int trial = 0; trial < 100; trial++) {
        unsigned char *p = (unsigned char *)w;
        for (size_t i = 0; i < NB * sizeof(*w); i++)
            p[i] = (unsigned char)rnd();
        for (int b = 0; b < NB; b++)
            w[b].d = janas_fp32_to_fp16(rndf(-0.01f, 0.01f));
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }
        janas_iq3s_dequantize(w, wf, N);
        janas_dequantize(JANAS_IQ3_S, w, w2, N);
        CHECK(memcmp(wf, w2, N * sizeof(float)) == 0,
              "iq3s: janas_dequantize differs from its unpacker");
        double expect = 0.0, mag = 0.0;
        for (int b = 0; b < NB; b++)
            for (int i = 0; i < JANAS_QK; i++) {
                double e = (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];
                expect += e;
                mag += fabs(e);
            }
        float ref = janas_iq3s_dot_ref(w, x, NB);
        CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "iq3s ref %g vs %g", ref,
              expect);
        CHECK(janas_dot(JANAS_IQ3_S, w, x, NB) == ref, "iq3s fast vs ref");
        for (int nv = 1; nv <= NV; nv++) {
            float out[NV];
            janas_dot_multi(JANAS_IQ3_S, w, x, NB, (size_t)nv, NB, out, 1);
            for (int v = 0; v < nv; v++)
                CHECK(out[v] == janas_iq3s_dot_ref(w, x + v * NB, NB),
                      "iq3s multi nv %d v %d", nv, v);
        }
    }
    free(w);
    free(x);
    free(xf);
    free(wf);
    free(w2);
}

static void test_dot_iq3xxs(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    struct janas_block_iq3xxs *w = malloc(NB * sizeof(*w));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float)),
          *w2 = malloc(N * sizeof(float));
    for (int trial = 0; trial < 100; trial++) {
        unsigned char *p = (unsigned char *)w;
        for (size_t i = 0; i < NB * sizeof(*w); i++)
            p[i] = (unsigned char)rnd();
        for (int b = 0; b < NB; b++)
            w[b].d = janas_fp32_to_fp16(rndf(-0.01f, 0.01f));
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }
        janas_iq3xxs_dequantize(w, wf, N);
        janas_dequantize(JANAS_IQ3_XXS, w, w2, N);
        CHECK(memcmp(wf, w2, N * sizeof(float)) == 0,
              "iq3xxs: janas_dequantize differs from its unpacker");
        double expect = 0.0, mag = 0.0;
        for (int b = 0; b < NB; b++)
            for (int i = 0; i < JANAS_QK; i++) {
                double e = (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];
                expect += e;
                mag += fabs(e);
            }
        float ref = janas_iq3xxs_dot_ref(w, x, NB);
        CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "iq3xxs ref %g vs %g",
              ref, expect);
        CHECK(janas_dot(JANAS_IQ3_XXS, w, x, NB) == ref, "iq3xxs fast vs ref");
        for (int nv = 1; nv <= NV; nv++) {
            float out[NV];
            janas_dot_multi(JANAS_IQ3_XXS, w, x, NB, (size_t)nv, NB, out, 1);
            for (int v = 0; v < nv; v++)
                CHECK(out[v] == janas_iq3xxs_dot_ref(w, x + v * NB, NB),
                      "iq3xxs multi nv %d v %d", nv, v);
        }
    }
    free(w);
    free(x);
    free(xf);
    free(wf);
    free(w2);
}

/* IQ2_XXS, IQ2_XS, IQ2_S: the reference against a float product over the
   dequantized weights, the kernels (single, 1-17 vectors, rows) equal to it
   to the bit */
static void test_dot_iq2(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    static const int types[] = {JANAS_IQ2_XXS, JANAS_IQ2_XS, JANAS_IQ2_S};
    static const char *const names[] = {"iq2_xxs", "iq2_xs", "iq2_s"};
    size_t big = NB * janas_qtype_block_size(JANAS_IQ2_S);
    unsigned char *w = malloc(big);
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float));
    float rows[4];
    for (int trial = 0; trial < 100; trial++)
        for (int k = 0; k < 3; k++) {
            int t = types[k];
            size_t bs = janas_qtype_block_size(t);
            for (size_t i = 0; i < NB * bs; i++)
                w[i] = (unsigned char)rnd();
            for (int b = 0; b < NB; b++) {
                uint16_t d = janas_fp32_to_fp16(rndf(-0.01f, 0.01f));
                memcpy(w + b * bs, &d, 2);
            }
            for (int v = 0; v < NV; v++) {
                for (int i = 0; i < N; i++)
                    xf[i] = rndf(-1.0f, 1.0f);
                janas_q8k_quantize(xf, x + v * NB, N);
            }
            janas_dequantize(t, w, wf, N);
            double expect = 0.0, mag = 0.0;
            for (int b = 0; b < NB; b++)
                for (int i = 0; i < JANAS_QK; i++) {
                    double e =
                        (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];
                    expect += e;
                    mag += fabs(e);
                }
            float ref = janas_iq2_dot_ref(t, w, x, NB);
            CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "%s ref %g vs %g",
                  names[k], ref, expect);
            CHECK(janas_dot(t, w, x, NB) == ref, "%s fast vs ref", names[k]);
            for (int nv = 1; nv <= NV; nv++) {
                float out[NV];
                janas_dot_multi(t, w, x, NB, (size_t)nv, NB, out, 1);
                for (int v = 0; v < nv; v++)
                    CHECK(out[v] == janas_iq2_dot_ref(t, w, x + v * NB, NB),
                          "%s multi nv %d v %d", names[k], nv, v);
            }
            janas_dot_rows(t, w, 2 * bs, x, 2, 4, rows);
            for (int r = 0; r < 4; r++)
                CHECK(rows[r] == janas_iq2_dot_ref(t, w + 2 * bs * r, x, 2),
                      "%s rows %d", names[k], r);
        }
    free(w);
    free(x);
    free(xf);
    free(wf);
}

static void test_dot_iq1m(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    struct janas_block_iq1m *w = malloc(NB * sizeof(*w));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float)),
          *w2 = malloc(N * sizeof(float));
    for (int trial = 0; trial < 100; trial++) {
        unsigned char *p = (unsigned char *)w;
        for (size_t i = 0; i < NB * sizeof(*w); i++)
            p[i] = (unsigned char)rnd();
        for (int b = 0; b < NB; b++) {
            /* a finite d in the scale words' top nibbles */
            uint16_t d = janas_fp32_to_fp16(rndf(-0.01f, 0.01f)), sc[4];
            memcpy(sc, w[b].scales, 8);
            for (int k = 0; k < 4; k++)
                sc[k] =
                    (uint16_t)((sc[k] & 0x0fff) | ((d >> 4 * k & 15) << 12));
            memcpy(w[b].scales, sc, 8);
        }
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }
        janas_iq1m_dequantize(w, wf, N);
        janas_dequantize(JANAS_IQ1_M, w, w2, N);
        CHECK(memcmp(wf, w2, N * sizeof(float)) == 0,
              "iq1m: janas_dequantize differs from its unpacker");
        double expect = 0.0, mag = 0.0;
        for (int b = 0; b < NB; b++)
            for (int i = 0; i < JANAS_QK; i++) {
                double e = (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];
                expect += e;
                mag += fabs(e);
            }
        float ref = janas_iq1m_dot_ref(w, x, NB);
        CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "iq1m ref %g vs %g", ref,
              expect);
        CHECK(janas_dot(JANAS_IQ1_M, w, x, NB) == ref, "iq1m fast vs ref");
        for (int nv = 1; nv <= NV; nv++) {
            float out[NV];
            janas_dot_multi(JANAS_IQ1_M, w, x, NB, (size_t)nv, NB, out, 1);
            for (int v = 0; v < nv; v++)
                CHECK(out[v] == janas_iq1m_dot_ref(w, x + v * NB, NB),
                      "iq1m multi nv %d v %d", nv, v);
        }
    }
    free(w);
    free(x);
    free(xf);
    free(wf);
    free(w2);
}

static void test_dot_iq1s(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    struct janas_block_iq1s *w = malloc(NB * sizeof(*w));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float)),
          *w2 = malloc(N * sizeof(float));
    for (int trial = 0; trial < 100; trial++) {
        unsigned char *p = (unsigned char *)w;
        for (size_t i = 0; i < NB * sizeof(*w); i++)
            p[i] = (unsigned char)rnd();
        for (int b = 0; b < NB; b++)
            w[b].d = janas_fp32_to_fp16(rndf(-0.01f, 0.01f));
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }
        janas_iq1s_dequantize(w, wf, N);
        janas_dequantize(JANAS_IQ1_S, w, w2, N);
        CHECK(memcmp(wf, w2, N * sizeof(float)) == 0,
              "iq1s: janas_dequantize differs from its unpacker");
        double expect = 0.0, mag = 0.0;
        for (int b = 0; b < NB; b++)
            for (int i = 0; i < JANAS_QK; i++) {
                double e = (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];
                expect += e;
                mag += fabs(e);
            }
        float ref = janas_iq1s_dot_ref(w, x, NB);
        CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "iq1s ref %g vs %g", ref,
              expect);
        CHECK(janas_dot(JANAS_IQ1_S, w, x, NB) == ref, "iq1s fast vs ref");
        for (int nv = 1; nv <= NV; nv++) {
            float out[NV];
            janas_dot_multi(JANAS_IQ1_S, w, x, NB, (size_t)nv, NB, out, 1);
            for (int v = 0; v < nv; v++)
                CHECK(out[v] == janas_iq1s_dot_ref(w, x + v * NB, NB),
                      "iq1s multi nv %d v %d", nv, v);
        }
    }
    free(w);
    free(x);
    free(xf);
    free(wf);
    free(w2);
}

static void test_dot_q2k(void)
{
    enum { NB = 8, N = NB * JANAS_QK, NV = 17 };
    struct janas_block_q2k *w = malloc(NB * sizeof(*w));
    struct janas_block_q8k *x = malloc(NV * NB * sizeof(*x));
    float *xf = malloc(N * sizeof(float)), *wf = malloc(N * sizeof(float)),
          *w2 = malloc(N * sizeof(float));
    for (int trial = 0; trial < 100; trial++) {
        unsigned char *p = (unsigned char *)w;
        for (size_t i = 0; i < NB * sizeof(*w); i++)
            p[i] = (unsigned char)rnd();
        for (int b = 0; b < NB; b++) {
            w[b].d = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
            w[b].dmin = janas_fp32_to_fp16(rndf(0.0001f, 0.01f));
        }
        for (int v = 0; v < NV; v++) {
            for (int i = 0; i < N; i++)
                xf[i] = rndf(-1.0f, 1.0f);
            janas_q8k_quantize(xf, x + v * NB, N);
        }
        janas_q2k_dequantize(w, wf, N);
        janas_dequantize(JANAS_Q2_K, w, w2, N);
        CHECK(memcmp(wf, w2, N * sizeof(float)) == 0,
              "q2k: janas_dequantize differs from its unpacker");
        double expect = 0.0, mag = 0.0;
        for (int b = 0; b < NB; b++)
            for (int i = 0; i < JANAS_QK; i++) {
                double e = (double)wf[b * JANAS_QK + i] * x[b].d * x[b].qs[i];
                expect += e;
                mag += fabs(e);
            }
        float ref = janas_q2k_dot_ref(w, x, NB);
        CHECK(fabs(ref - expect) <= 1e-5 * mag + 1e-6, "q2k ref %g vs %g", ref,
              expect);
        CHECK(janas_dot(JANAS_Q2_K, w, x, NB) == ref, "q2k fast vs ref");
        for (int nv = 1; nv <= NV; nv++) {
            float out[NV];
            janas_dot_multi(JANAS_Q2_K, w, x, NB, (size_t)nv, NB, out, 1);
            for (int v = 0; v < nv; v++)
                CHECK(out[v] == janas_q2k_dot_ref(w, x + v * NB, NB),
                      "q2k multi nv %d v %d", nv, v);
        }
    }
    free(w);
    free(x);
    free(xf);
    free(wf);
    free(w2);
}

int main(void)
{
    /* a test that deadlocks must fail, not hold the suite for ever */
    alarm(600);
    test_fp16();
    test_dot();
    test_dot_q6k();
    test_q6k_planes();
    test_q6kpn_axpy();
    test_q6kpn_chunk();
    test_matvec();
    test_matvec_group();
    test_matvec_multi();
    test_dot_multi();
    test_det();
    test_dot_q5k_q8_0();
    test_dot_iq4();
    test_dot_mxfp4();
    test_dot_q4_0_q5_1();
    test_dot_q5_0_q2_0();
    test_multi_rows_long();
    test_q6k_tiles();
    test_row_tails();
    test_dot_q3k();
    test_dot_iq3s();
    test_dot_iq3xxs();
    test_dot_iq2();
    test_dot_iq1m();
    test_dot_iq1s();
    test_dot_q2k();
    test_quantize_q4k();
    test_quantize_q6k();
    test_quantize_q8k();
    test_dequantize();
    test_q8w();
    test_bf16_rows();
    printf("test_quant: kernel %s, %s\n", janas_q4k_kernel_name(),
           failures ? "FAILED" : "ok");
    return failures ? 1 : 0;
}