Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

34.2 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * quant.h - quantized block formats and dot-product kernels.
 *
 * The block layouts are byte-compatible with the GGUF k-quants, so weights can
 * be taken from existing GGUF files and results compared against llama.cpp:
 *
 *   Q4_K: 256 weights in 144 bytes. Two fp16 super-block scales (d, dmin),
 *         12 bytes packing eight 6-bit sub-block scales and eight 6-bit
 *         sub-block mins, and 128 bytes of 4-bit values. Weight i of
 *         sub-block s is  d * scale[s] * q - dmin * min[s].
 *         The 128 value bytes hold four 64-weight chunks of 32 bytes: in chunk
 *         c the low nibbles are sub-block 2c and the high nibbles 2c+1.
 *
 *   Q6_K: 256 weights in 210 bytes: low 4 bits (128 bytes), high 2 bits (64
 *         bytes), sixteen int8 scales and one fp16 scale d. Weight i is
 *         d * scale[i / 16] * (q - 32), q in 0..63.
 *
 *   Q8_K: activations, 256 values as int8 with one float scale, plus the sums
 *         of each group of 16 values (needed for the mins term).
 */
#ifndef JANAS_LLM_QUANT_H
#define JANAS_LLM_QUANT_H

#include <stddef.h>
#include <stdint.h>

#define JANAS_QK 256 /* weights per super-block */

struct janas_block_q4k {
    uint16_t d;    /* fp16 */
    uint16_t dmin; /* fp16 */
    uint8_t scales[12];
    uint8_t qs[JANAS_QK / 2];
};

struct janas_block_q5k {
    uint16_t d;    /* fp16 */
    uint16_t dmin; /* fp16 */
    uint8_t scales[12];
    uint8_t qh[JANAS_QK / 8]; /* bit s of qh[l]: 5th bit of weight 32 s + l */
    uint8_t qs[JANAS_QK / 2]; /* low 4 bits, as in Q4_K */
};

/* Q8_0: 32 weights and their scale; eight of them cover a Q8_K block. */
struct janas_block_q8_0 {
    uint16_t d; /* fp16 */
    int8_t qs[32];
};

/*
 * IQ4_NL: 32 weights, each a 4-bit index into janas_iq4_values (sixteen
 * levels spaced unevenly, closer near zero), times the block's scale;
 * eight blocks cover a Q8_K block. Byte l holds weight l in its low nibble
 * and weight l + 16 in its high one.
 */
struct janas_block_iq4nl {
    uint16_t d; /* fp16 */
    uint8_t qs[16];
};

/*
 * IQ4_XS: 256 weights in eight groups of 32, each group laid out as an
 * IQ4_NL block without its scale. Group g's scale is d * (ls - 32), ls a
 * 6-bit number: its low four bits in scales_l[g / 2] (low nibble for even
 * g), its high two in bits 2g, 2g + 1 of scales_h.
 */
struct janas_block_iq4xs {
    uint16_t d; /* fp16 */
    uint16_t scales_h;
    uint8_t scales_l[JANAS_QK / 64];
    uint8_t qs[JANAS_QK / 2];
};

/* The sixteen levels of IQ4_NL and IQ4_XS. */
extern const int8_t janas_iq4_values[16];

/* Q4_0: IQ4_NL's layout with evenly spaced levels, -8 to 7 (the kernels
   are IQ4_NL's with another table). */
struct janas_block_q4_0 {
    uint16_t d; /* fp16 */
    uint8_t qs[16];
};
extern const int8_t janas_q4_0_values[16];

/*
 * MXFP4 (ggml's type 39, the OCP microscaling format): 32 weights, each a
 * 4-bit E2M1 code into janas_mxfp4_values (0, 1, 2, 3, 4, 6, 8, 12 and
 * their opposites: twice the format's values), times 2^(e - 128), e the
 * block's E8M0 byte. IQ4_NL's layout with that byte for a scale: eight
 * blocks cover a Q8_K block, byte l holds weight l low and l + 16 high.
 */
struct janas_block_mxfp4 {
    uint8_t e; /* E8M0: the power of two, 127 the unit */
    uint8_t qs[16];
};
extern const int8_t janas_mxfp4_values[16];
/* 2^(e - 128) exactly, as ggml's GGML_E8M0_TO_FP32_HALF (subnormal below
   e = 2): an MXFP4 block's scale */
float janas_e8m0_half(uint8_t e);

/*
 * NVFP4 (ggml's type 40, NVIDIA's 4-bit format): 64 weights in four
 * sub-blocks of 16, each a 4-bit E2M1 code into janas_mxfp4_values (twice
 * the format's values) times its sub-block's UE4M3 scale (four exponent
 * bits, bias 7, three of mantissa; 0 and 0x7F read as zero). Sub-block s
 * keeps its codes in qs[8 s .. 8 s + 7]: byte j holds weight 16 s + j low
 * and 16 s + j + 8 high. Four blocks cover a Q8_K block.
 */
struct janas_block_nvfp4 {
    uint8_t d[4]; /* UE4M3 */
    uint8_t qs[32];
};
/* A UE4M3 scale times one half, as ggml's ggml_ue4m3_to_fp32 (the half
   pairs with janas_mxfp4_values' doubled values) */
float janas_ue4m3_half(uint8_t x);

/*
 * Q5_1: 32 weights d * q + m, q of five bits: the low four as in Q4_0,
 * the fifth of weight l in bit l of qh (little-endian).
 */
struct janas_block_q5_1 {
    uint16_t d, m; /* fp16 */
    uint8_t qh[4];
    uint8_t qs[16];
};

/*
 * Q3_K: 256 weights of three bits in sixteen groups of 16, each with a
 * 6-bit scale (sc - 32; the twelve bytes hold the low four bits of scales
 * 0-7 and 8-15 in the low and high nibbles of bytes 0-7, the top two bits
 * in bytes 8-11). Weight e = 128 n + 32 j + 16 h + l: its low two bits are
 * bits 2j of qs[32 n + 16 h + l], its third bit j + 4n of hmask[16 h + l],
 * and it is those three bits less 4 when the third is clear: -4 to 3.
 */
struct janas_block_q3k {
    uint8_t hmask[JANAS_QK / 8];
    uint8_t qs[JANAS_QK / 4];
    uint8_t scales[12];
    uint16_t d; /* fp16 */
};

/*
 * IQ3_S: 256 weights in eight groups of 32. Group g: eight 9-bit indices
 * into a grid of 512 entries of four levels each (qs[8g + k], its ninth
 * bit bit k of qh[g]), the signs of its 32 weights in signs[4g .. 4g + 3]
 * (bit e for weight e, set: negative), and the scale 1 + 2 x a nibble of
 * scales[g / 2] (low for even g), times d.
 */
struct janas_block_iq3s {
    uint16_t d; /* fp16 */
    uint8_t qs[JANAS_QK / 4];
    uint8_t qh[JANAS_QK / 32];
    uint8_t signs[JANAS_QK / 8];
    uint8_t scales[JANAS_QK / 64];
};

/*
 * IQ3_XXS: 256 weights in eight groups of 32, 98 bytes. Group g: eight 8-bit
 * indices into a grid of 256 entries of four magnitudes each (qs[8g + k],
 * weights 4k to 4k + 3), then a 32-bit word at qs[64 + 4g]: four 7-bit
 * indices of sign bytes (bits 7k, for weights 8k to 8k + 7; the eighth sign
 * makes the count of negative ones even) and the scale s in its top four
 * bits. The weight is d (1 + 2 s) / 4 times the signed magnitude.
 */
struct janas_block_iq3xxs {
    uint16_t d; /* fp16 */
    uint8_t qs[3 * JANAS_QK / 8];
};

/*
 * IQ2_XXS, IQ2_XS, IQ2_S: 256 weights in 32 sub-groups of eight, each an
 * entry of a grid (eight magnitudes) and a sign byte (bit j for weight j),
 * a scale s for every one or two of them; the weight is d (1 + 2 s) / 8
 * times the signed magnitude. IQ2_XXS (66 bytes): per group of 32 two
 * words, four 8-bit indices into a grid of 256, then four 7-bit indices of
 * sign bytes (the eighth sign making the count of negative ones even) and
 * the group's scale in the top four bits. IQ2_XS (74 bytes): per
 * sub-group a 16-bit word, a 9-bit index into a grid of 512 and a 7-bit
 * index of a sign byte; scales[g]: the scales of sub-groups 4g, 4g + 1
 * (low nibble) and 4g + 2, 4g + 3. IQ2_S (82 bytes): qs[4g + k] the low
 * eight bits of a 10-bit index into a grid of 1024 (the high two in bits
 * 2k of qh[g]), qs[32 + 4g + k] its sign byte as it is, scales as IQ2_XS.
 */
struct janas_block_iq2xxs {
    uint16_t d; /* fp16 */
    uint16_t qs[JANAS_QK / 8];
};
struct janas_block_iq2xs {
    uint16_t d; /* fp16 */
    uint16_t qs[JANAS_QK / 8];
    uint8_t scales[JANAS_QK / 32];
};
struct janas_block_iq2s {
    uint16_t d; /* fp16 */
    uint8_t qs[JANAS_QK / 4];
    uint8_t qh[JANAS_QK / 32];
    uint8_t scales[JANAS_QK / 32];
};

/*
 * IQ1_M: 256 weights in 32 sub-groups of eight, 56 bytes: each an 11-bit
 * index into a grid of 2048 entries of eight levels -1, 0, 1 (qs[4g + k]
 * the low eight bits; the high three in bits 0-2 or 4-6 of qh[2g + k / 2],
 * for k even or odd) plus a shift of +1/8, or -1/8 where bit 3 or 7 of that
 * byte is set; a 3-bit scale s for each half of a group (the 16-bit word
 * g / 2 of scales, bits 6 (g % 2) and 3 more), the weight d (2 s + 1)
 * (level + shift). d is an fp16 whose four nibbles are the top four bits
 * of the four scale words.
 */
/* IQ1_S (50 bytes): IQ1_M's grid with one scale and one shift per group of
   32: qh[g] holds the high three bits of its four indices (bits 3k), the
   scale s in bits 12-14 and the shift's sign in bit 15; d an fp16 first. */
struct janas_block_iq1s {
    uint16_t d; /* fp16 */
    uint8_t qs[JANAS_QK / 8];
    uint16_t qh[JANAS_QK / 32];
};
struct janas_block_iq1m {
    uint8_t qs[JANAS_QK / 8];
    uint8_t qh[JANAS_QK / 16];
    uint8_t scales[JANAS_QK / 32];
};

/*
 * Q2_K: 256 weights in sixteen groups of 16, 84 bytes. scales[g]: group g's
 * scale (low nibble) and min (high); qs: in each half of the block (32
 * bytes) bits 2j of byte l are weight 32 j + l of that half, l < 32. Weight
 * i of group g is d * scale * q - dmin * min, q in 0..3.
 */
struct janas_block_q2k {
    uint8_t scales[JANAS_QK / 16];
    uint8_t qs[JANAS_QK / 4];
    uint16_t d, dmin; /* fp16 */
};

/* Q4_1: Q5_1 without the fifth bits, q of four (0-15). */
struct janas_block_q4_1 {
    uint16_t d, m; /* fp16 */
    uint8_t qs[16];
};

/* Q5_0: Q5_1 without the minimum, the weight d * (q - 16). */
struct janas_block_q5_0 {
    uint16_t d; /* fp16 */
    uint8_t qh[4];
    uint8_t qs[16];
};

/*
 * Q2_0 (ggml's type 42): 64 weights d * (q - 1), q of two bits, so -1 to 2:
 * weight i in bits 2 (i % 4) of qs[i / 4]. Four blocks cover a Q8_K block,
 * each two of its blocks of 32.
 */
struct janas_block_q2_0 {
    uint16_t d; /* fp16 */
    uint8_t qs[16];
};

struct janas_block_q6k {
    uint8_t ql[JANAS_QK / 2];
    uint8_t qh[JANAS_QK / 4];
    int8_t scales[JANAS_QK / 16];
    uint16_t d; /* fp16 */
};

/*
 * Q6_K in bit planes: the six bits of every weight cut into three planes of
 * two bits, each packed exactly as Q6_K's qh field (byte 32n + l, bits 2p).
 * The high plane carries the scales, the other two are bare. Weight i is
 * 16 * p2 + 4 * p1 + p0, so a row can be read one plane at a time: with all
 * three the value is the Q6_K one, with two the centre of the four it stands
 * for (4 * q + 1.5), with one the centre of sixteen (16 * q + 7.5).
 */
struct janas_block_q6kp {
    uint8_t q[JANAS_QK / 4]; /* the high two bits of every weight */
    int8_t scales[JANAS_QK / 16];
    uint16_t d; /* fp16 */
};
#define JANAS_Q6KP_PLANE (JANAS_QK / 4) /* bytes of one bare plane, 64 */

struct janas_block_q8k {
    float d;
    int8_t qs[JANAS_QK];
    int16_t bsums[JANAS_QK / 16];
    /* the same 256 activations in eight blocks of 32, each with a scale of
       its own, for the weights stored in blocks of 32 (Q8_0, Q4_0, Q4_1,
       Q5_1, IQ4_NL), as llama.cpp gives them: one scale for 256 lost the
       small activations beside a large one, and Gemma-4-26B, all Q8_0 and
       Q5_1 where it is most sensitive, strayed from the exact result half
       again as far as llama.cpp did (27 Sep 2026: the likeliest token at
       147 of 200 positions against 160) */
    float d32[8];
    int8_t qs32[JANAS_QK];
    int16_t bsums32[JANAS_QK / 16];
    /* bsums two by two: the sums of 32, which the Q4_K and Q5_K products
       weigh by their mins, made once here instead of for every row */
    int16_t bsums2[JANAS_QK / 32];
};

_Static_assert(sizeof(struct janas_block_q4k) == 144, "q4k block size");
_Static_assert(sizeof(struct janas_block_q6k) == 210, "q6k block size");
_Static_assert(sizeof(struct janas_block_q6kp) == 82, "q6kp block size");
_Static_assert(sizeof(struct janas_block_q5k) == 176, "q5k block size");
_Static_assert(sizeof(struct janas_block_q8_0) == 34, "q8_0 block size");
_Static_assert(sizeof(struct janas_block_iq4nl) == 18, "iq4_nl block size");
_Static_assert(sizeof(struct janas_block_mxfp4) == 17, "mxfp4 block size");
_Static_assert(sizeof(struct janas_block_iq4xs) == 136, "iq4_xs block size");
_Static_assert(sizeof(struct janas_block_q4_0) == 18, "q4_0 block size");
_Static_assert(sizeof(struct janas_block_q5_1) == 24, "q5_1 block size");
_Static_assert(sizeof(struct janas_block_q4_1) == 20, "q4_1 block size");
_Static_assert(sizeof(struct janas_block_q5_0) == 22, "q5_0 block size");
_Static_assert(sizeof(struct janas_block_q2_0) == 18, "q2_0 block size");
_Static_assert(sizeof(struct janas_block_nvfp4) == 36, "nvfp4 block size");
_Static_assert(sizeof(struct janas_block_iq3xxs) == 98, "iq3_xxs block size");
_Static_assert(sizeof(struct janas_block_iq2xxs) == 66, "iq2_xxs block size");
_Static_assert(sizeof(struct janas_block_iq2xs) == 74, "iq2_xs block size");
_Static_assert(sizeof(struct janas_block_iq2s) == 82, "iq2_s block size");
_Static_assert(sizeof(struct janas_block_iq1m) == 56, "iq1_m block size");
_Static_assert(sizeof(struct janas_block_iq1s) == 50, "iq1_s block size");
_Static_assert(sizeof(struct janas_block_q2k) == 84, "q2_k block size");
_Static_assert(sizeof(struct janas_block_q3k) == 110, "q3k block size");
_Static_assert(sizeof(struct janas_block_iq3s) == 110, "iq3_s block size");
_Static_assert(sizeof(struct janas_block_q8k) == 628, "q8k block size");

/* Quantization types, numbered as in GGUF (ggml_type). */
enum janas_qtype {
    JANAS_Q4_0 = 2, /* sizes per 256 weights: 8 blocks, 144 bytes */
    JANAS_Q4_1 = 3, /* 8 blocks, 160 bytes */
    JANAS_Q5_0 = 6, /* 8 blocks, 176 bytes */
    JANAS_Q5_1 = 7, /* 8 blocks, 192 bytes */
    JANAS_Q8_0 = 8,
    JANAS_Q2_K = 10, /* 84 bytes per 256 weights */
    JANAS_Q3_K = 11, /* sizes counted per 256 weights: 8 blocks, 272 bytes */
    JANAS_Q4_K = 12,
    JANAS_Q5_K = 13,
    JANAS_Q6_K = 14,
    JANAS_IQ2_XXS = 16, /* 66 bytes per 256 weights */
    JANAS_IQ2_XS = 17,  /* 74 */
    JANAS_IQ3_XXS = 18, /* 98 */
    JANAS_IQ4_NL = 20,
    JANAS_IQ3_S = 21, /* sizes per 256 weights: 8 blocks, 144 bytes */
    JANAS_IQ2_S = 22, /* 82 bytes per 256 weights */
    JANAS_IQ1_S = 19, /* 50 bytes per 256 weights */
    JANAS_IQ1_M = 29, /* 56 */
    JANAS_IQ4_XS = 23,
    JANAS_MXFP4 = 39, /* sizes per 256 weights: 8 blocks, 136 bytes */
    JANAS_NVFP4 = 40, /* 4 blocks of 64, 144 bytes */
    JANAS_Q2_0 = 42,  /* 4 blocks of 64, 72 bytes */
    /* Janas' own, outside the GGUF numbering: the high plane of a Q6_K row
       cut into bit planes. Its block size counts that plane alone; the other
       two are JANAS_Q6KP_PLANE bytes each and are passed separately. */
    JANAS_Q6_K_P = 214,
    /* Janas' own too: a down matrix by neuron, its rows the neurons (each
       its outputs, of the model's width) stored by block of 256 outputs -
       every neuron's block 0, then every neuron's block 1 - in Q6_K's bit
       planes laid out as JANAS_Q6_K_P's, after a float per output channel
       that multiplies the sum (tools/jns_neurons.c): read a neuron at a time,
       at the depth its contribution earns (janas_q6kpn_axpy). The block size
       counts the base plane alone, as JANAS_Q6_K_P's; the matrix's bytes add
       the channels' floats. */
    JANAS_Q6_K_PN = 215
};

/* Bytes of one 256-weight block of the type, 0 if unsupported. */
size_t janas_qtype_block_size(int type);

/*
 * A row of n weights with n no multiple of JANAS_QK, as the 640 of
 * Qwen3.8-Flash-Next's experts: only in the types stored in blocks of 32 or
 * 64, its bytes the n / JANAS_QK whole groups, then the tail's blocks. Its
 * product is the whole groups' as janas_dot gives it, plus the tail's as
 * janas_dot gives the tail padded with zero blocks to a whole group (no
 * whole group: the tail's alone); the activations are (n + JANAS_QK - 1) /
 * JANAS_QK Q8_K blocks, those past n zero. A row whose n is a multiple of
 * JANAS_QK is janas_dot's, janas_dot_rows', janas_dot_multi's.
 */
/* The bytes of a row of n weights of the type, 0 if it cannot end at n. */
size_t janas_qtype_row_bytes(int type, size_t n);
float janas_dot_n(int type, const void *w, const struct janas_block_q8k *x,
                  size_t n);
void janas_dot_rows_n(int type, const void *w, size_t row_bytes,
                      const struct janas_block_q8k *x, size_t n, size_t rows,
                      float *y);
void janas_dot_multi_n(int type, const void *w, const struct janas_block_q8k *x,
                       size_t x_stride, size_t nv, size_t n, float *out,
                       size_t out_stride);
/* janas_dot_multi_n on rows rows of row_bytes: y[r + v * y_stride], the
   same bits */
void janas_dot_multi_rows_n(int type, const void *w, size_t row_bytes,
                            const struct janas_block_q8k *x, size_t x_stride,
                            size_t nv, size_t n, size_t rows, float *y,
                            size_t y_stride);

/*
 * Dot product of nb blocks of a supported type with nb Q8_K blocks. Aborts on
 * an unsupported type: check janas_qtype_block_size() first.
 */
float janas_dot(int type, const void *w, const struct janas_block_q8k *x,
                size_t nb);

/*
 * One row of n weights (n a multiple of JANAS_QK) of a supported type into
 * floats. Aborts on an unsupported type, bit planes among them (which
 * janas_qtype_block_size() still sizes: the loaders refuse them for any
 * resident tensor). Every caller goes through here, so a type
 * the engine reads in its products is read the same way when a row of it has
 * to be unpacked whole - a token embedding, say.
 */
void janas_dequantize(int type, const void *w, float *y, size_t n);

/*
 * The dot products of `rows` consecutive weight rows (row r at w + r *
 * row_bytes) with one vector: y[r]. The same bits as janas_dot row by row,
 * with the type chosen once.
 */
void janas_dot_rows(int type, const void *w, size_t row_bytes,
                    const struct janas_block_q8k *x, size_t nb, size_t rows,
                    float *y);

/*
 * The dot products of one weight row with nv vectors (vector v at x + v *
 * x_stride blocks), result v at out[v * out_stride]. Bit-identical to calling
 * janas_dot for each vector, but unpacks the weights once per four vectors.
 */
void janas_dot_multi(int type, const void *w, const struct janas_block_q8k *x,
                     size_t x_stride, size_t nv, size_t nb, float *out,
                     size_t out_stride);

/* janas_dot_multi on rows rows of row_bytes from w: y[r + v * y_stride].
   The same bits; Q2_0 reorders the vectors' activations once for all the
   rows. */
void janas_dot_multi_rows(int type, const void *w, size_t row_bytes,
                          const struct janas_block_q8k *x, size_t x_stride,
                          size_t nv, size_t nb, size_t rows, float *y,
                          size_t y_stride);

/* Float helpers, AVX2 when available: dot product and y += a * x. */
float janas_dot_f32(const float *a, const float *b, size_t n);
void janas_axpy_f32(float *y, float a, const float *x, size_t n);

float janas_fp16_to_fp32(uint16_t h);
uint16_t janas_fp32_to_fp16(float f);

/* MXFP4 (ggml's type 39): a block of 32 weights is an E8M0 scale byte e and
   16 bytes of 4-bit codes, the weight kv[code] * 2^(e - 128) with kv the 16
   integers 0, 1, 2, 3, 4, 6, 8, 12 and their opposites; codes j and j + 16
   share byte j, low and high half. */
#define JANAS_MXFP4_BLOCK 17

/* n MXFP4 blocks as Q8_0 blocks, exactly: the codes' integers as the
   weights, the power of two as the fp16 scale. A block whose power is below
   fp16's least, 2^-24 (weights under 3.6e-7), is written as zeros and
   counted in *tiny. -1 for a scale beyond fp16 (2^16 and more, or E8M0's
   NaN), which no model's weights have. */
int janas_mxfp4_to_q8_0(const uint8_t *in, struct janas_block_q8_0 *out,
                        size_t n, size_t *tiny);

/* Quantizes n floats (n multiple of JANAS_QK) into n / JANAS_QK blocks. */
void janas_q8k_quantize(const float *x, struct janas_block_q8k *y, size_t n);

/* The same with deterministic operations only (no division): what the
   GPU reproduces bit for bit, for values a GPU may also compute. */
void janas_q8k_quantize_det(const float *x, struct janas_block_q8k *y,
                            size_t n);

/*
 * Quantizes n floats (n multiple of JANAS_QK) into Q4_K blocks: per 32-value
 * sub-block an affine fit searched over shrunken ranges and refit by least
 * squares, then the two block factors refit by least squares against the
 * 6-bit sub-block scales. For offline conversion, not for the hot path.
 */
void janas_q4k_quantize(const float *x, struct janas_block_q4k *y, size_t n);
/* The same with each sub-block's range taken as it is, not searched: nine
   times faster, for requantizing at load (Qwen3.5-9B's output head, a
   billion weights: 2.5 s against 22 s to open), and on that head the mean
   logit difference against llama.cpp 0.157 where 0.154 (7 Oct 2026). */
void janas_q4k_quantize_fast(const float *x, struct janas_block_q4k *y,
                             size_t n);

/* Dequantizes n weights (n multiple of JANAS_QK). */
void janas_q4k_dequantize(const struct janas_block_q4k *x, float *y, size_t n);

/*
 * Dot product of nb Q4_K blocks with nb Q8_K blocks. janas_q4k_dot uses the
 * fastest kernel the CPU supports, chosen once at run time; the _ref variant
 * is the portable scalar definition every other kernel is tested against.
 */
float janas_q4k_dot(const struct janas_block_q4k *w,
                    const struct janas_block_q8k *x, size_t nb);
float janas_q4k_dot_ref(const struct janas_block_q4k *w,
                        const struct janas_block_q8k *x, size_t nb);

/* Name of the kernel janas_q4k_dot dispatches to ("avx2" or "ref"). */
const char *janas_q4k_kernel_name(void);

/* Whether the AVX-VNNI kernels are in use: the CPU has them and
   JANAS_KERNELS does not ask for plain AVX2. Attention asks the same
   question, and both answers give the same numbers. */
int janas_use_vnni(void);

/* The same three entry points for Q6_K. */
void janas_q6k_dequantize(const struct janas_block_q6k *x, float *y, size_t n);
float janas_q6k_dot(const struct janas_block_q6k *w,
                    const struct janas_block_q8k *x, size_t nb);
float janas_q6k_dot_ref(const struct janas_block_q6k *w,
                        const struct janas_block_q8k *x, size_t nb);

/*
 * The bit-plane form of Q6_K. p1 and p0 are the bare planes of the same
 * blocks (JANAS_Q6KP_PLANE bytes each); p0 may be NULL, and p1 only when p0
 * is too, giving the three levels 6, 4 and 2 bits per weight. With all three
 * planes the result is bit-identical to janas_q6k_dot on the block the three
 * came from.
 */
void janas_q6k_to_planes(const struct janas_block_q6k *src, size_t nb,
                         struct janas_block_q6kp *base, uint8_t *p1,
                         uint8_t *p0);
void janas_q6kp_to_q6k(const struct janas_block_q6kp *base, const uint8_t *p1,
                       const uint8_t *p0, size_t nb,
                       struct janas_block_q6k *dst);
void janas_q6kp_dequantize(const struct janas_block_q6kp *base,
                           const uint8_t *p1, const uint8_t *p0, float *y,
                           size_t n);
/* n floats (n a multiple of JANAS_QK) as Q6_K: each group of 16 its scale,
   chosen among a few near its largest magnitude for the least squared
   error, the block's factor then fitted by least squares (quant_q6k.c). */
void janas_q6k_quantize(const float *x, struct janas_block_q6k *y, size_t n);
/* y[0 .. n) += h times a row of n weights in Q6_K's bit planes read at a
   level - 3 the three planes, 2 the base and p1, 1 the base alone; the
   planes below the level may be NULL - the same values as
   janas_q6kp_dequantize: a row of JANAS_Q6_K_PN, one neuron's outputs. */
void janas_q6kpn_axpy(const struct janas_block_q6kp *base, const uint8_t *p1,
                      const uint8_t *p0, int level, float h, float *y,
                      size_t n);
/* The same without vector instructions, bit for bit: the reference. */
void janas_q6kpn_axpy_ref(const struct janas_block_q6kp *base,
                          const uint8_t *p1, const uint8_t *p0, int level,
                          float h, float *y, size_t n);
/* The down by neuron in integers: n <= JANAS_PN_CHUNK neurons of a matrix
   of `rows` neurons stored by block of outputs (block b of neuron j at
   base + b * rows + j, its planes at the same index of p1 and p0), neuron
   i = js[i] read at level lv[i] (1 to 3) with activation h[i]; their block
   b of 256 outputs added to y[0 .. 256). The factors h d sc are
   quantized to eight bits over the chunk, as the products by rows quantize
   their activations, and the sums are exact integers: janas_q6kpn_chunk_ref,
   without vector instructions, gives the same bits. */
#define JANAS_PN_CHUNK 128
void janas_q6kpn_chunk(const struct janas_block_q6kp *base, const uint8_t *p1,
                       const uint8_t *p0, size_t rows, const uint32_t *js,
                       const uint8_t *lv, const float *h, uint32_t n, size_t b,
                       float *y);
void janas_q6kpn_chunk_ref(const struct janas_block_q6kp *base,
                           const uint8_t *p1, const uint8_t *p0, size_t rows,
                           const uint32_t *js, const uint8_t *lv,
                           const float *h, uint32_t n, size_t b, float *y);
float janas_q6kp_dot(const struct janas_block_q6kp *w, const uint8_t *p1,
                     const uint8_t *p0, const struct janas_block_q8k *x,
                     size_t nb);
float janas_q6kp_dot_ref(const struct janas_block_q6kp *w, const uint8_t *p1,
                         const uint8_t *p0, const struct janas_block_q8k *x,
                         size_t nb);
/* Q6_K or Q3_K (p1, p0 NULL) or Q6_K_P (each row's planes pb bytes apart from
   p1 and p0, NULL past its level) rows against nv vectors, y[r + v ys], eight
   rows at a time as tiles (quant_dotn.h q6k_gemm): the rows done, a
   multiple of eight (0 where the tiles do not serve: few vectors, no
   AVX2, JANAS_CPU_TILES=0). The same bits as janas_dot_multi. */
size_t janas_q6k_tile_rows(int type, const void *w, size_t row_bytes,
                           const uint8_t *p1, const uint8_t *p0, size_t pb,
                           const struct janas_block_q8k *x, size_t x_stride,
                           size_t nv, size_t nb, size_t rows, float *y,
                           size_t y_stride);
/* As janas_dot_multi, for the plane form: one row against nv vectors. */
void janas_q6kp_dot_multi(const struct janas_block_q6kp *w, const uint8_t *p1,
                          const uint8_t *p0, const struct janas_block_q8k *x,
                          size_t x_stride, size_t nv, size_t nb, float *out,
                          size_t out_stride);

/* And for Q5_K (nb blocks of 256 weights) and Q8_0 (nb groups of eight
   32-weight blocks, 256 weights each). Q8_0 sums each of the eight block
   positions in its own accumulator (one fma per block), then adds the eight
   as janas_hsum8 does: ((a0+a4)+(a2+a6)) + ((a1+a5)+(a3+a7)). */
void janas_q5k_dequantize(const struct janas_block_q5k *x, float *y, size_t n);
float janas_q5k_dot(const struct janas_block_q5k *w,
                    const struct janas_block_q8k *x, size_t nb);
float janas_q5k_dot_ref(const struct janas_block_q5k *w,
                        const struct janas_block_q8k *x, size_t nb);
void janas_q8_0_dequantize(const struct janas_block_q8_0 *x, float *y,
                           size_t n);
/* n floats (n a multiple of 32) into Q8_0 blocks as llama.cpp's
   quantize_row_q8_0_ref makes them: d = amax / 127, weights rounded half
   away from zero */
void janas_q8_0_quantize(const float *x, struct janas_block_q8_0 *y, size_t n);
/* IQ4_NL (nb groups of eight blocks, as Q8_0: each block's sum in its own
   accumulator, the eight added as janas_hsum8 does) and IQ4_XS (one exact
   integer sum per 256 weights, scales included, then one fma: as Q4_K). */
void janas_iq4nl_dequantize(const struct janas_block_iq4nl *x, float *y,
                            size_t n);
float janas_iq4nl_dot(const struct janas_block_iq4nl *w,
                      const struct janas_block_q8k *x, size_t nb);
float janas_iq4nl_dot_ref(const struct janas_block_iq4nl *w,
                          const struct janas_block_q8k *x, size_t nb);
/* MXFP4: IQ4_NL's products, the block's power of two for its scale */
void janas_mxfp4_dequantize(const struct janas_block_mxfp4 *x, float *y,
                            size_t n);
float janas_mxfp4_dot(const struct janas_block_mxfp4 *w,
                      const struct janas_block_q8k *x, size_t nb);
float janas_mxfp4_dot_ref(const struct janas_block_mxfp4 *w,
                          const struct janas_block_q8k *x, size_t nb);
/* NVFP4: as llama.cpp's product against Q8_0, sub-block by sub-block: a
   lane per block of 32 activations, each sub-block's exact sum times its
   scale and the block's */
void janas_nvfp4_dequantize(const struct janas_block_nvfp4 *x, float *y,
                            size_t n);
float janas_nvfp4_dot(const struct janas_block_nvfp4 *w,
                      const struct janas_block_q8k *x, size_t nb);
float janas_nvfp4_dot_ref(const struct janas_block_nvfp4 *w,
                          const struct janas_block_q8k *x, size_t nb);
void janas_iq4xs_dequantize(const struct janas_block_iq4xs *x, float *y,
                            size_t n);
float janas_iq4xs_dot(const struct janas_block_iq4xs *w,
                      const struct janas_block_q8k *x, size_t nb);
float janas_iq4xs_dot_ref(const struct janas_block_iq4xs *w,
                          const struct janas_block_q8k *x, size_t nb);
/* Q4_0 (as IQ4_NL) and Q5_1 (per block position, a lane for d x the
   weights' sums and one for m x the activations' sums, from the Q8_K
   block sums; each set of eight added as janas_hsum8 does, then the two) */
void janas_q4_0_dequantize(const struct janas_block_q4_0 *x, float *y,
                           size_t n);
float janas_q4_0_dot(const struct janas_block_q4_0 *w,
                     const struct janas_block_q8k *x, size_t nb);
float janas_q4_0_dot_ref(const struct janas_block_q4_0 *w,
                         const struct janas_block_q8k *x, size_t nb);
void janas_q5_1_dequantize(const struct janas_block_q5_1 *x, float *y,
                           size_t n);
float janas_q5_1_dot(const struct janas_block_q5_1 *w,
                     const struct janas_block_q8k *x, size_t nb);
float janas_q5_1_dot_ref(const struct janas_block_q5_1 *w,
                         const struct janas_block_q8k *x, size_t nb);
/* Q3_K: one exact integer sum per 256 weights, scales included, then one
   fma (as Q4_K) */
void janas_q3k_dequantize(const struct janas_block_q3k *x, float *y, size_t n);
float janas_q3k_dot(const struct janas_block_q3k *w,
                    const struct janas_block_q8k *x, size_t nb);
float janas_q3k_dot_ref(const struct janas_block_q3k *w,
                        const struct janas_block_q8k *x, size_t nb);
/* IQ3_S: one exact integer sum per 256 weights, then one fma */
void janas_iq3s_dequantize(const struct janas_block_iq3s *x, float *y,
                           size_t n);
float janas_iq3s_dot(const struct janas_block_iq3s *w,
                     const struct janas_block_q8k *x, size_t nb);
float janas_iq3s_dot_ref(const struct janas_block_iq3s *w,
                         const struct janas_block_q8k *x, size_t nb);
/* Q4_1: as Q5_1 */
void janas_q4_1_dequantize(const struct janas_block_q4_1 *x, float *y,
                           size_t n);
float janas_q4_1_dot(const struct janas_block_q4_1 *w,
                     const struct janas_block_q8k *x, size_t nb);
float janas_q4_1_dot_ref(const struct janas_block_q4_1 *w,
                         const struct janas_block_q8k *x, size_t nb);
/* IQ2_XXS, IQ2_XS, IQ2_S: one exact integer sum per 256 weights, scales
   included, then one fma, as IQ3_S; w is nb blocks of the type */
void janas_iq2_dequantize(int type, const void *x, float *y, size_t n);
float janas_iq2_dot(int type, const void *w, const struct janas_block_q8k *x,
                    size_t nb);
float janas_iq2_dot_ref(int type, const void *w,
                        const struct janas_block_q8k *x, size_t nb);
/* Q2_K: Q4_K's scheme, an exact sum for the scales and one for the mins,
   then an fma each */
void janas_q2k_dequantize(const struct janas_block_q2k *x, float *y, size_t n);
float janas_q2k_dot(const struct janas_block_q2k *w,
                    const struct janas_block_q8k *x, size_t nb);
float janas_q2k_dot_ref(const struct janas_block_q2k *w,
                        const struct janas_block_q8k *x, size_t nb);
/* IQ1_M: (8 level + shift sign) times the half's 2 s + 1, one exact integer
   sum per 256 weights, then one fma with d / 8 */
void janas_iq1m_dequantize(const struct janas_block_iq1m *x, float *y,
                           size_t n);
void janas_iq1s_dequantize(const struct janas_block_iq1s *x, float *y,
                           size_t n);
float janas_iq1s_dot(const struct janas_block_iq1s *w,
                     const struct janas_block_q8k *x, size_t nb);
float janas_iq1s_dot_ref(const struct janas_block_iq1s *w,
                         const struct janas_block_q8k *x, size_t nb);
float janas_iq1m_dot(const struct janas_block_iq1m *w,
                     const struct janas_block_q8k *x, size_t nb);
float janas_iq1m_dot_ref(const struct janas_block_iq1m *w,
                         const struct janas_block_q8k *x, size_t nb);
/* IQ3_XXS: one exact integer sum per 256 weights, scales included, then
   one fma, as IQ3_S */
void janas_iq3xxs_dequantize(const struct janas_block_iq3xxs *x, float *y,
                             size_t n);
float janas_iq3xxs_dot(const struct janas_block_iq3xxs *w,
                       const struct janas_block_q8k *x, size_t nb);
float janas_iq3xxs_dot_ref(const struct janas_block_iq3xxs *w,
                           const struct janas_block_q8k *x, size_t nb);
/* Q5_0 (nb groups of eight blocks): per block position the exact sum of
   (q - 16) x a in a lane of its own, the eight added as janas_hsum8 does */
void janas_q5_0_dequantize(const struct janas_block_q5_0 *x, float *y,
                           size_t n);
float janas_q5_0_dot(const struct janas_block_q5_0 *w,
                     const struct janas_block_q8k *x, size_t nb);
float janas_q5_0_dot_ref(const struct janas_block_q5_0 *w,
                         const struct janas_block_q8k *x, size_t nb);
/* Q2_0 (nb groups of four blocks): a lane for each block of 32
   activations, the exact sum of (q - 1) x a times d and their scale, the
   eight added as janas_hsum8 does */
void janas_q2_0_dequantize(const struct janas_block_q2_0 *x, float *y,
                           size_t n);
float janas_q2_0_dot(const struct janas_block_q2_0 *w,
                     const struct janas_block_q8k *x, size_t nb);
float janas_q2_0_dot_ref(const struct janas_block_q2_0 *w,
                         const struct janas_block_q8k *x, size_t nb);
float janas_q8_0_dot(const struct janas_block_q8_0 *w,
                     const struct janas_block_q8k *x, size_t nb);
float janas_q8_0_dot_ref(const struct janas_block_q8_0 *w,
                         const struct janas_block_q8k *x, size_t nb);

#endif