37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* quant.h - quantized block formats and dot-product kernels.
*
* The block layouts are byte-compatible with the GGUF k-quants, so weights can
* be taken from existing GGUF files and results compared against llama.cpp:
*
* Q4_K: 256 weights in 144 bytes. Two fp16 super-block scales (d, dmin),
* 12 bytes packing eight 6-bit sub-block scales and eight 6-bit
* sub-block mins, and 128 bytes of 4-bit values. Weight i of
* sub-block s is d * scale[s] * q - dmin * min[s].
* The 128 value bytes hold four 64-weight chunks of 32 bytes: in chunk
* c the low nibbles are sub-block 2c and the high nibbles 2c+1.
*
* Q6_K: 256 weights in 210 bytes: low 4 bits (128 bytes), high 2 bits (64
* bytes), sixteen int8 scales and one fp16 scale d. Weight i is
* d * scale[i / 16] * (q - 32), q in 0..63.
*
* Q8_K: activations, 256 values as int8 with one float scale, plus the sums
* of each group of 16 values (needed for the mins term).
*/
#ifndef JANAS_LLM_QUANT_H
#define JANAS_LLM_QUANT_H
#include <stddef.h>
#include <stdint.h>
#define JANAS_QK 256 /* weights per super-block */
struct janas_block_q4k {
uint16_t d; /* fp16 */
uint16_t dmin; /* fp16 */
uint8_t scales[12];
uint8_t qs[JANAS_QK / 2];
};
struct janas_block_q5k {
uint16_t d; /* fp16 */
uint16_t dmin; /* fp16 */
uint8_t scales[12];
uint8_t qh[JANAS_QK / 8]; /* bit s of qh[l]: 5th bit of weight 32 s + l */
uint8_t qs[JANAS_QK / 2]; /* low 4 bits, as in Q4_K */
};
/* Q8_0: 32 weights and their scale; eight of them cover a Q8_K block. */
struct janas_block_q8_0 {
uint16_t d; /* fp16 */
int8_t qs[32];
};
/*
* IQ4_NL: 32 weights, each a 4-bit index into janas_iq4_values (sixteen
* levels spaced unevenly, closer near zero), times the block's scale;
* eight blocks cover a Q8_K block. Byte l holds weight l in its low nibble
* and weight l + 16 in its high one.
*/
struct janas_block_iq4nl {
uint16_t d; /* fp16 */
uint8_t qs[16];
};
/*
* IQ4_XS: 256 weights in eight groups of 32, each group laid out as an
* IQ4_NL block without its scale. Group g's scale is d * (ls - 32), ls a
* 6-bit number: its low four bits in scales_l[g / 2] (low nibble for even
* g), its high two in bits 2g, 2g + 1 of scales_h.
*/
struct janas_block_iq4xs {
uint16_t d; /* fp16 */
uint16_t scales_h;
uint8_t scales_l[JANAS_QK / 64];
uint8_t qs[JANAS_QK / 2];
};
/* The sixteen levels of IQ4_NL and IQ4_XS. */
extern const int8_t janas_iq4_values[16];
/* Q4_0: IQ4_NL's layout with evenly spaced levels, -8 to 7 (the kernels
are IQ4_NL's with another table). */
struct janas_block_q4_0 {
uint16_t d; /* fp16 */
uint8_t qs[16];
};
extern const int8_t janas_q4_0_values[16];
/*
* MXFP4 (ggml's type 39, the OCP microscaling format): 32 weights, each a
* 4-bit E2M1 code into janas_mxfp4_values (0, 1, 2, 3, 4, 6, 8, 12 and
* their opposites: twice the format's values), times 2^(e - 128), e the
* block's E8M0 byte. IQ4_NL's layout with that byte for a scale: eight
* blocks cover a Q8_K block, byte l holds weight l low and l + 16 high.
*/
struct janas_block_mxfp4 {
uint8_t e; /* E8M0: the power of two, 127 the unit */
uint8_t qs[16];
};
extern const int8_t janas_mxfp4_values[16];
/* 2^(e - 128) exactly, as ggml's GGML_E8M0_TO_FP32_HALF (subnormal below
e = 2): an MXFP4 block's scale */
float janas_e8m0_half(uint8_t e);
/*
* NVFP4 (ggml's type 40, NVIDIA's 4-bit format): 64 weights in four
* sub-blocks of 16, each a 4-bit E2M1 code into janas_mxfp4_values (twice
* the format's values) times its sub-block's UE4M3 scale (four exponent
* bits, bias 7, three of mantissa; 0 and 0x7F read as zero). Sub-block s
* keeps its codes in qs[8 s .. 8 s + 7]: byte j holds weight 16 s + j low
* and 16 s + j + 8 high. Four blocks cover a Q8_K block.
*/
struct janas_block_nvfp4 {
uint8_t d[4]; /* UE4M3 */
uint8_t qs[32];
};
/* A UE4M3 scale times one half, as ggml's ggml_ue4m3_to_fp32 (the half
pairs with janas_mxfp4_values' doubled values) */
float janas_ue4m3_half(uint8_t x);
/*
* Q5_1: 32 weights d * q + m, q of five bits: the low four as in Q4_0,
* the fifth of weight l in bit l of qh (little-endian).
*/
struct janas_block_q5_1 {
uint16_t d, m; /* fp16 */
uint8_t qh[4];
uint8_t qs[16];
};
/*
* Q3_K: 256 weights of three bits in sixteen groups of 16, each with a
* 6-bit scale (sc - 32; the twelve bytes hold the low four bits of scales
* 0-7 and 8-15 in the low and high nibbles of bytes 0-7, the top two bits
* in bytes 8-11). Weight e = 128 n + 32 j + 16 h + l: its low two bits are
* bits 2j of qs[32 n + 16 h + l], its third bit j + 4n of hmask[16 h + l],
* and it is those three bits less 4 when the third is clear: -4 to 3.
*/
struct janas_block_q3k {
uint8_t hmask[JANAS_QK / 8];
uint8_t qs[JANAS_QK / 4];
uint8_t scales[12];
uint16_t d; /* fp16 */
};
/*
* IQ3_S: 256 weights in eight groups of 32. Group g: eight 9-bit indices
* into a grid of 512 entries of four levels each (qs[8g + k], its ninth
* bit bit k of qh[g]), the signs of its 32 weights in signs[4g .. 4g + 3]
* (bit e for weight e, set: negative), and the scale 1 + 2 x a nibble of
* scales[g / 2] (low for even g), times d.
*/
struct janas_block_iq3s {
uint16_t d; /* fp16 */
uint8_t qs[JANAS_QK / 4];
uint8_t qh[JANAS_QK / 32];
uint8_t signs[JANAS_QK / 8];
uint8_t scales[JANAS_QK / 64];
};
/*
* IQ3_XXS: 256 weights in eight groups of 32, 98 bytes. Group g: eight 8-bit
* indices into a grid of 256 entries of four magnitudes each (qs[8g + k],
* weights 4k to 4k + 3), then a 32-bit word at qs[64 + 4g]: four 7-bit
* indices of sign bytes (bits 7k, for weights 8k to 8k + 7; the eighth sign
* makes the count of negative ones even) and the scale s in its top four
* bits. The weight is d (1 + 2 s) / 4 times the signed magnitude.
*/
struct janas_block_iq3xxs {
uint16_t d; /* fp16 */
uint8_t qs[3 * JANAS_QK / 8];
};
/*
* IQ2_XXS, IQ2_XS, IQ2_S: 256 weights in 32 sub-groups of eight, each an
* entry of a grid (eight magnitudes) and a sign byte (bit j for weight j),
* a scale s for every one or two of them; the weight is d (1 + 2 s) / 8
* times the signed magnitude. IQ2_XXS (66 bytes): per group of 32 two
* words, four 8-bit indices into a grid of 256, then four 7-bit indices of
* sign bytes (the eighth sign making the count of negative ones even) and
* the group's scale in the top four bits. IQ2_XS (74 bytes): per
* sub-group a 16-bit word, a 9-bit index into a grid of 512 and a 7-bit
* index of a sign byte; scales[g]: the scales of sub-groups 4g, 4g + 1
* (low nibble) and 4g + 2, 4g + 3. IQ2_S (82 bytes): qs[4g + k] the low
* eight bits of a 10-bit index into a grid of 1024 (the high two in bits
* 2k of qh[g]), qs[32 + 4g + k] its sign byte as it is, scales as IQ2_XS.
*/
struct janas_block_iq2xxs {
uint16_t d; /* fp16 */
uint16_t qs[JANAS_QK / 8];
};
struct janas_block_iq2xs {
uint16_t d; /* fp16 */
uint16_t qs[JANAS_QK / 8];
uint8_t scales[JANAS_QK / 32];
};
struct janas_block_iq2s {
uint16_t d; /* fp16 */
uint8_t qs[JANAS_QK / 4];
uint8_t qh[JANAS_QK / 32];
uint8_t scales[JANAS_QK / 32];
};
/*
* IQ1_M: 256 weights in 32 sub-groups of eight, 56 bytes: each an 11-bit
* index into a grid of 2048 entries of eight levels -1, 0, 1 (qs[4g + k]
* the low eight bits; the high three in bits 0-2 or 4-6 of qh[2g + k / 2],
* for k even or odd) plus a shift of +1/8, or -1/8 where bit 3 or 7 of that
* byte is set; a 3-bit scale s for each half of a group (the 16-bit word
* g / 2 of scales, bits 6 (g % 2) and 3 more), the weight d (2 s + 1)
* (level + shift). d is an fp16 whose four nibbles are the top four bits
* of the four scale words.
*/
/* IQ1_S (50 bytes): IQ1_M's grid with one scale and one shift per group of
32: qh[g] holds the high three bits of its four indices (bits 3k), the
scale s in bits 12-14 and the shift's sign in bit 15; d an fp16 first. */
struct janas_block_iq1s {
uint16_t d; /* fp16 */
uint8_t qs[JANAS_QK / 8];
uint16_t qh[JANAS_QK / 32];
};
struct janas_block_iq1m {
uint8_t qs[JANAS_QK / 8];
uint8_t qh[JANAS_QK / 16];
uint8_t scales[JANAS_QK / 32];
};
/*
* Q2_K: 256 weights in sixteen groups of 16, 84 bytes. scales[g]: group g's
* scale (low nibble) and min (high); qs: in each half of the block (32
* bytes) bits 2j of byte l are weight 32 j + l of that half, l < 32. Weight
* i of group g is d * scale * q - dmin * min, q in 0..3.
*/
struct janas_block_q2k {
uint8_t scales[JANAS_QK / 16];
uint8_t qs[JANAS_QK / 4];
uint16_t d, dmin; /* fp16 */
};
/* Q4_1: Q5_1 without the fifth bits, q of four (0-15). */
struct janas_block_q4_1 {
uint16_t d, m; /* fp16 */
uint8_t qs[16];
};
/* Q5_0: Q5_1 without the minimum, the weight d * (q - 16). */
struct janas_block_q5_0 {
uint16_t d; /* fp16 */
uint8_t qh[4];
uint8_t qs[16];
};
/*
* Q2_0 (ggml's type 42): 64 weights d * (q - 1), q of two bits, so -1 to 2:
* weight i in bits 2 (i % 4) of qs[i / 4]. Four blocks cover a Q8_K block,
* each two of its blocks of 32.
*/
struct janas_block_q2_0 {
uint16_t d; /* fp16 */
uint8_t qs[16];
};
struct janas_block_q6k {
uint8_t ql[JANAS_QK / 2];
uint8_t qh[JANAS_QK / 4];
int8_t scales[JANAS_QK / 16];
uint16_t d; /* fp16 */
};
/*
* Q6_K in bit planes: the six bits of every weight cut into three planes of
* two bits, each packed exactly as Q6_K's qh field (byte 32n + l, bits 2p).
* The high plane carries the scales, the other two are bare. Weight i is
* 16 * p2 + 4 * p1 + p0, so a row can be read one plane at a time: with all
* three the value is the Q6_K one, with two the centre of the four it stands
* for (4 * q + 1.5), with one the centre of sixteen (16 * q + 7.5).
*/
struct janas_block_q6kp {
uint8_t q[JANAS_QK / 4]; /* the high two bits of every weight */
int8_t scales[JANAS_QK / 16];
uint16_t d; /* fp16 */
};
#define JANAS_Q6KP_PLANE (JANAS_QK / 4) /* bytes of one bare plane, 64 */
struct janas_block_q8k {
float d;
int8_t qs[JANAS_QK];
int16_t bsums[JANAS_QK / 16];
/* the same 256 activations in eight blocks of 32, each with a scale of
its own, for the weights stored in blocks of 32 (Q8_0, Q4_0, Q4_1,
Q5_1, IQ4_NL), as llama.cpp gives them: one scale for 256 lost the
small activations beside a large one, and Gemma-4-26B, all Q8_0 and
Q5_1 where it is most sensitive, strayed from the exact result half
again as far as llama.cpp did (27 Sep 2026: the likeliest token at
147 of 200 positions against 160) */
float d32[8];
int8_t qs32[JANAS_QK];
int16_t bsums32[JANAS_QK / 16];
/* bsums two by two: the sums of 32, which the Q4_K and Q5_K products
weigh by their mins, made once here instead of for every row */
int16_t bsums2[JANAS_QK / 32];
};
_Static_assert(sizeof(struct janas_block_q4k) == 144, "q4k block size");
_Static_assert(sizeof(struct janas_block_q6k) == 210, "q6k block size");
_Static_assert(sizeof(struct janas_block_q6kp) == 82, "q6kp block size");
_Static_assert(sizeof(struct janas_block_q5k) == 176, "q5k block size");
_Static_assert(sizeof(struct janas_block_q8_0) == 34, "q8_0 block size");
_Static_assert(sizeof(struct janas_block_iq4nl) == 18, "iq4_nl block size");
_Static_assert(sizeof(struct janas_block_mxfp4) == 17, "mxfp4 block size");
_Static_assert(sizeof(struct janas_block_iq4xs) == 136, "iq4_xs block size");
_Static_assert(sizeof(struct janas_block_q4_0) == 18, "q4_0 block size");
_Static_assert(sizeof(struct janas_block_q5_1) == 24, "q5_1 block size");
_Static_assert(sizeof(struct janas_block_q4_1) == 20, "q4_1 block size");
_Static_assert(sizeof(struct janas_block_q5_0) == 22, "q5_0 block size");
_Static_assert(sizeof(struct janas_block_q2_0) == 18, "q2_0 block size");
_Static_assert(sizeof(struct janas_block_nvfp4) == 36, "nvfp4 block size");
_Static_assert(sizeof(struct janas_block_iq3xxs) == 98, "iq3_xxs block size");
_Static_assert(sizeof(struct janas_block_iq2xxs) == 66, "iq2_xxs block size");
_Static_assert(sizeof(struct janas_block_iq2xs) == 74, "iq2_xs block size");
_Static_assert(sizeof(struct janas_block_iq2s) == 82, "iq2_s block size");
_Static_assert(sizeof(struct janas_block_iq1m) == 56, "iq1_m block size");
_Static_assert(sizeof(struct janas_block_iq1s) == 50, "iq1_s block size");
_Static_assert(sizeof(struct janas_block_q2k) == 84, "q2_k block size");
_Static_assert(sizeof(struct janas_block_q3k) == 110, "q3k block size");
_Static_assert(sizeof(struct janas_block_iq3s) == 110, "iq3_s block size");
_Static_assert(sizeof(struct janas_block_q8k) == 628, "q8k block size");
/* Quantization types, numbered as in GGUF (ggml_type). */
enum janas_qtype {
JANAS_Q4_0 = 2, /* sizes per 256 weights: 8 blocks, 144 bytes */
JANAS_Q4_1 = 3, /* 8 blocks, 160 bytes */
JANAS_Q5_0 = 6, /* 8 blocks, 176 bytes */
JANAS_Q5_1 = 7, /* 8 blocks, 192 bytes */
JANAS_Q8_0 = 8,
JANAS_Q2_K = 10, /* 84 bytes per 256 weights */
JANAS_Q3_K = 11, /* sizes counted per 256 weights: 8 blocks, 272 bytes */
JANAS_Q4_K = 12,
JANAS_Q5_K = 13,
JANAS_Q6_K = 14,
JANAS_IQ2_XXS = 16, /* 66 bytes per 256 weights */
JANAS_IQ2_XS = 17, /* 74 */
JANAS_IQ3_XXS = 18, /* 98 */
JANAS_IQ4_NL = 20,
JANAS_IQ3_S = 21, /* sizes per 256 weights: 8 blocks, 144 bytes */
JANAS_IQ2_S = 22, /* 82 bytes per 256 weights */
JANAS_IQ1_S = 19, /* 50 bytes per 256 weights */
JANAS_IQ1_M = 29, /* 56 */
JANAS_IQ4_XS = 23,
JANAS_MXFP4 = 39, /* sizes per 256 weights: 8 blocks, 136 bytes */
JANAS_NVFP4 = 40, /* 4 blocks of 64, 144 bytes */
JANAS_Q2_0 = 42, /* 4 blocks of 64, 72 bytes */
/* Janas' own, outside the GGUF numbering: the high plane of a Q6_K row
cut into bit planes. Its block size counts that plane alone; the other
two are JANAS_Q6KP_PLANE bytes each and are passed separately. */
JANAS_Q6_K_P = 214,
/* Janas' own too: a down matrix by neuron, its rows the neurons (each
its outputs, of the model's width) stored by block of 256 outputs -
every neuron's block 0, then every neuron's block 1 - in Q6_K's bit
planes laid out as JANAS_Q6_K_P's, after a float per output channel
that multiplies the sum (tools/jns_neurons.c): read a neuron at a time,
at the depth its contribution earns (janas_q6kpn_axpy). The block size
counts the base plane alone, as JANAS_Q6_K_P's; the matrix's bytes add
the channels' floats. */
JANAS_Q6_K_PN = 215
};
/* Bytes of one 256-weight block of the type, 0 if unsupported. */
size_t janas_qtype_block_size(int type);
/*
* A row of n weights with n no multiple of JANAS_QK, as the 640 of
* Qwen3.8-Flash-Next's experts: only in the types stored in blocks of 32 or
* 64, its bytes the n / JANAS_QK whole groups, then the tail's blocks. Its
* product is the whole groups' as janas_dot gives it, plus the tail's as
* janas_dot gives the tail padded with zero blocks to a whole group (no
* whole group: the tail's alone); the activations are (n + JANAS_QK - 1) /
* JANAS_QK Q8_K blocks, those past n zero. A row whose n is a multiple of
* JANAS_QK is janas_dot's, janas_dot_rows', janas_dot_multi's.
*/
/* The bytes of a row of n weights of the type, 0 if it cannot end at n. */
size_t janas_qtype_row_bytes(int type, size_t n);
float janas_dot_n(int type, const void *w, const struct janas_block_q8k *x,
size_t n);
void janas_dot_rows_n(int type, const void *w, size_t row_bytes,
const struct janas_block_q8k *x, size_t n, size_t rows,
float *y);
void janas_dot_multi_n(int type, const void *w, const struct janas_block_q8k *x,
size_t x_stride, size_t nv, size_t n, float *out,
size_t out_stride);
/* janas_dot_multi_n on rows rows of row_bytes: y[r + v * y_stride], the
same bits */
void janas_dot_multi_rows_n(int type, const void *w, size_t row_bytes,
const struct janas_block_q8k *x, size_t x_stride,
size_t nv, size_t n, size_t rows, float *y,
size_t y_stride);
/*
* Dot product of nb blocks of a supported type with nb Q8_K blocks. Aborts on
* an unsupported type: check janas_qtype_block_size() first.
*/
float janas_dot(int type, const void *w, const struct janas_block_q8k *x,
size_t nb);
/*
* One row of n weights (n a multiple of JANAS_QK) of a supported type into
* floats. Aborts on an unsupported type, bit planes among them (which
* janas_qtype_block_size() still sizes: the loaders refuse them for any
* resident tensor). Every caller goes through here, so a type
* the engine reads in its products is read the same way when a row of it has
* to be unpacked whole - a token embedding, say.
*/
void janas_dequantize(int type, const void *w, float *y, size_t n);
/*
* The dot products of `rows` consecutive weight rows (row r at w + r *
* row_bytes) with one vector: y[r]. The same bits as janas_dot row by row,
* with the type chosen once.
*/
void janas_dot_rows(int type, const void *w, size_t row_bytes,
const struct janas_block_q8k *x, size_t nb, size_t rows,
float *y);
/*
* The dot products of one weight row with nv vectors (vector v at x + v *
* x_stride blocks), result v at out[v * out_stride]. Bit-identical to calling
* janas_dot for each vector, but unpacks the weights once per four vectors.
*/
void janas_dot_multi(int type, const void *w, const struct janas_block_q8k *x,
size_t x_stride, size_t nv, size_t nb, float *out,
size_t out_stride);
/* janas_dot_multi on rows rows of row_bytes from w: y[r + v * y_stride].
The same bits; Q2_0 reorders the vectors' activations once for all the
rows. */
void janas_dot_multi_rows(int type, const void *w, size_t row_bytes,
const struct janas_block_q8k *x, size_t x_stride,
size_t nv, size_t nb, size_t rows, float *y,
size_t y_stride);
/* Float helpers, AVX2 when available: dot product and y += a * x. */
float janas_dot_f32(const float *a, const float *b, size_t n);
void janas_axpy_f32(float *y, float a, const float *x, size_t n);
float janas_fp16_to_fp32(uint16_t h);
uint16_t janas_fp32_to_fp16(float f);
/* MXFP4 (ggml's type 39): a block of 32 weights is an E8M0 scale byte e and
16 bytes of 4-bit codes, the weight kv[code] * 2^(e - 128) with kv the 16
integers 0, 1, 2, 3, 4, 6, 8, 12 and their opposites; codes j and j + 16
share byte j, low and high half. */
#define JANAS_MXFP4_BLOCK 17
/* n MXFP4 blocks as Q8_0 blocks, exactly: the codes' integers as the
weights, the power of two as the fp16 scale. A block whose power is below
fp16's least, 2^-24 (weights under 3.6e-7), is written as zeros and
counted in *tiny. -1 for a scale beyond fp16 (2^16 and more, or E8M0's
NaN), which no model's weights have. */
int janas_mxfp4_to_q8_0(const uint8_t *in, struct janas_block_q8_0 *out,
size_t n, size_t *tiny);
/* Quantizes n floats (n multiple of JANAS_QK) into n / JANAS_QK blocks. */
void janas_q8k_quantize(const float *x, struct janas_block_q8k *y, size_t n);
/* The same with deterministic operations only (no division): what the
GPU reproduces bit for bit, for values a GPU may also compute. */
void janas_q8k_quantize_det(const float *x, struct janas_block_q8k *y,
size_t n);
/*
* Quantizes n floats (n multiple of JANAS_QK) into Q4_K blocks: per 32-value
* sub-block an affine fit searched over shrunken ranges and refit by least
* squares, then the two block factors refit by least squares against the
* 6-bit sub-block scales. For offline conversion, not for the hot path.
*/
void janas_q4k_quantize(const float *x, struct janas_block_q4k *y, size_t n);
/* The same with each sub-block's range taken as it is, not searched: nine
times faster, for requantizing at load (Qwen3.5-9B's output head, a
billion weights: 2.5 s against 22 s to open), and on that head the mean
logit difference against llama.cpp 0.157 where 0.154 (7 Oct 2026). */
void janas_q4k_quantize_fast(const float *x, struct janas_block_q4k *y,
size_t n);
/* Dequantizes n weights (n multiple of JANAS_QK). */
void janas_q4k_dequantize(const struct janas_block_q4k *x, float *y, size_t n);
/*
* Dot product of nb Q4_K blocks with nb Q8_K blocks. janas_q4k_dot uses the
* fastest kernel the CPU supports, chosen once at run time; the _ref variant
* is the portable scalar definition every other kernel is tested against.
*/
float janas_q4k_dot(const struct janas_block_q4k *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q4k_dot_ref(const struct janas_block_q4k *w,
const struct janas_block_q8k *x, size_t nb);
/* Name of the kernel janas_q4k_dot dispatches to ("avx2" or "ref"). */
const char *janas_q4k_kernel_name(void);
/* Whether the AVX-VNNI kernels are in use: the CPU has them and
JANAS_KERNELS does not ask for plain AVX2. Attention asks the same
question, and both answers give the same numbers. */
int janas_use_vnni(void);
/* The same three entry points for Q6_K. */
void janas_q6k_dequantize(const struct janas_block_q6k *x, float *y, size_t n);
float janas_q6k_dot(const struct janas_block_q6k *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q6k_dot_ref(const struct janas_block_q6k *w,
const struct janas_block_q8k *x, size_t nb);
/*
* The bit-plane form of Q6_K. p1 and p0 are the bare planes of the same
* blocks (JANAS_Q6KP_PLANE bytes each); p0 may be NULL, and p1 only when p0
* is too, giving the three levels 6, 4 and 2 bits per weight. With all three
* planes the result is bit-identical to janas_q6k_dot on the block the three
* came from.
*/
void janas_q6k_to_planes(const struct janas_block_q6k *src, size_t nb,
struct janas_block_q6kp *base, uint8_t *p1,
uint8_t *p0);
void janas_q6kp_to_q6k(const struct janas_block_q6kp *base, const uint8_t *p1,
const uint8_t *p0, size_t nb,
struct janas_block_q6k *dst);
void janas_q6kp_dequantize(const struct janas_block_q6kp *base,
const uint8_t *p1, const uint8_t *p0, float *y,
size_t n);
/* n floats (n a multiple of JANAS_QK) as Q6_K: each group of 16 its scale,
chosen among a few near its largest magnitude for the least squared
error, the block's factor then fitted by least squares (quant_q6k.c). */
void janas_q6k_quantize(const float *x, struct janas_block_q6k *y, size_t n);
/* y[0 .. n) += h times a row of n weights in Q6_K's bit planes read at a
level - 3 the three planes, 2 the base and p1, 1 the base alone; the
planes below the level may be NULL - the same values as
janas_q6kp_dequantize: a row of JANAS_Q6_K_PN, one neuron's outputs. */
void janas_q6kpn_axpy(const struct janas_block_q6kp *base, const uint8_t *p1,
const uint8_t *p0, int level, float h, float *y,
size_t n);
/* The same without vector instructions, bit for bit: the reference. */
void janas_q6kpn_axpy_ref(const struct janas_block_q6kp *base,
const uint8_t *p1, const uint8_t *p0, int level,
float h, float *y, size_t n);
/* The down by neuron in integers: n <= JANAS_PN_CHUNK neurons of a matrix
of `rows` neurons stored by block of outputs (block b of neuron j at
base + b * rows + j, its planes at the same index of p1 and p0), neuron
i = js[i] read at level lv[i] (1 to 3) with activation h[i]; their block
b of 256 outputs added to y[0 .. 256). The factors h d sc are
quantized to eight bits over the chunk, as the products by rows quantize
their activations, and the sums are exact integers: janas_q6kpn_chunk_ref,
without vector instructions, gives the same bits. */
#define JANAS_PN_CHUNK 128
void janas_q6kpn_chunk(const struct janas_block_q6kp *base, const uint8_t *p1,
const uint8_t *p0, size_t rows, const uint32_t *js,
const uint8_t *lv, const float *h, uint32_t n, size_t b,
float *y);
void janas_q6kpn_chunk_ref(const struct janas_block_q6kp *base,
const uint8_t *p1, const uint8_t *p0, size_t rows,
const uint32_t *js, const uint8_t *lv,
const float *h, uint32_t n, size_t b, float *y);
float janas_q6kp_dot(const struct janas_block_q6kp *w, const uint8_t *p1,
const uint8_t *p0, const struct janas_block_q8k *x,
size_t nb);
float janas_q6kp_dot_ref(const struct janas_block_q6kp *w, const uint8_t *p1,
const uint8_t *p0, const struct janas_block_q8k *x,
size_t nb);
/* Q6_K or Q3_K (p1, p0 NULL) or Q6_K_P (each row's planes pb bytes apart from
p1 and p0, NULL past its level) rows against nv vectors, y[r + v ys], eight
rows at a time as tiles (quant_dotn.h q6k_gemm): the rows done, a
multiple of eight (0 where the tiles do not serve: few vectors, no
AVX2, JANAS_CPU_TILES=0). The same bits as janas_dot_multi. */
size_t janas_q6k_tile_rows(int type, const void *w, size_t row_bytes,
const uint8_t *p1, const uint8_t *p0, size_t pb,
const struct janas_block_q8k *x, size_t x_stride,
size_t nv, size_t nb, size_t rows, float *y,
size_t y_stride);
/* As janas_dot_multi, for the plane form: one row against nv vectors. */
void janas_q6kp_dot_multi(const struct janas_block_q6kp *w, const uint8_t *p1,
const uint8_t *p0, const struct janas_block_q8k *x,
size_t x_stride, size_t nv, size_t nb, float *out,
size_t out_stride);
/* And for Q5_K (nb blocks of 256 weights) and Q8_0 (nb groups of eight
32-weight blocks, 256 weights each). Q8_0 sums each of the eight block
positions in its own accumulator (one fma per block), then adds the eight
as janas_hsum8 does: ((a0+a4)+(a2+a6)) + ((a1+a5)+(a3+a7)). */
void janas_q5k_dequantize(const struct janas_block_q5k *x, float *y, size_t n);
float janas_q5k_dot(const struct janas_block_q5k *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q5k_dot_ref(const struct janas_block_q5k *w,
const struct janas_block_q8k *x, size_t nb);
void janas_q8_0_dequantize(const struct janas_block_q8_0 *x, float *y,
size_t n);
/* n floats (n a multiple of 32) into Q8_0 blocks as llama.cpp's
quantize_row_q8_0_ref makes them: d = amax / 127, weights rounded half
away from zero */
void janas_q8_0_quantize(const float *x, struct janas_block_q8_0 *y, size_t n);
/* IQ4_NL (nb groups of eight blocks, as Q8_0: each block's sum in its own
accumulator, the eight added as janas_hsum8 does) and IQ4_XS (one exact
integer sum per 256 weights, scales included, then one fma: as Q4_K). */
void janas_iq4nl_dequantize(const struct janas_block_iq4nl *x, float *y,
size_t n);
float janas_iq4nl_dot(const struct janas_block_iq4nl *w,
const struct janas_block_q8k *x, size_t nb);
float janas_iq4nl_dot_ref(const struct janas_block_iq4nl *w,
const struct janas_block_q8k *x, size_t nb);
/* MXFP4: IQ4_NL's products, the block's power of two for its scale */
void janas_mxfp4_dequantize(const struct janas_block_mxfp4 *x, float *y,
size_t n);
float janas_mxfp4_dot(const struct janas_block_mxfp4 *w,
const struct janas_block_q8k *x, size_t nb);
float janas_mxfp4_dot_ref(const struct janas_block_mxfp4 *w,
const struct janas_block_q8k *x, size_t nb);
/* NVFP4: as llama.cpp's product against Q8_0, sub-block by sub-block: a
lane per block of 32 activations, each sub-block's exact sum times its
scale and the block's */
void janas_nvfp4_dequantize(const struct janas_block_nvfp4 *x, float *y,
size_t n);
float janas_nvfp4_dot(const struct janas_block_nvfp4 *w,
const struct janas_block_q8k *x, size_t nb);
float janas_nvfp4_dot_ref(const struct janas_block_nvfp4 *w,
const struct janas_block_q8k *x, size_t nb);
void janas_iq4xs_dequantize(const struct janas_block_iq4xs *x, float *y,
size_t n);
float janas_iq4xs_dot(const struct janas_block_iq4xs *w,
const struct janas_block_q8k *x, size_t nb);
float janas_iq4xs_dot_ref(const struct janas_block_iq4xs *w,
const struct janas_block_q8k *x, size_t nb);
/* Q4_0 (as IQ4_NL) and Q5_1 (per block position, a lane for d x the
weights' sums and one for m x the activations' sums, from the Q8_K
block sums; each set of eight added as janas_hsum8 does, then the two) */
void janas_q4_0_dequantize(const struct janas_block_q4_0 *x, float *y,
size_t n);
float janas_q4_0_dot(const struct janas_block_q4_0 *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q4_0_dot_ref(const struct janas_block_q4_0 *w,
const struct janas_block_q8k *x, size_t nb);
void janas_q5_1_dequantize(const struct janas_block_q5_1 *x, float *y,
size_t n);
float janas_q5_1_dot(const struct janas_block_q5_1 *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q5_1_dot_ref(const struct janas_block_q5_1 *w,
const struct janas_block_q8k *x, size_t nb);
/* Q3_K: one exact integer sum per 256 weights, scales included, then one
fma (as Q4_K) */
void janas_q3k_dequantize(const struct janas_block_q3k *x, float *y, size_t n);
float janas_q3k_dot(const struct janas_block_q3k *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q3k_dot_ref(const struct janas_block_q3k *w,
const struct janas_block_q8k *x, size_t nb);
/* IQ3_S: one exact integer sum per 256 weights, then one fma */
void janas_iq3s_dequantize(const struct janas_block_iq3s *x, float *y,
size_t n);
float janas_iq3s_dot(const struct janas_block_iq3s *w,
const struct janas_block_q8k *x, size_t nb);
float janas_iq3s_dot_ref(const struct janas_block_iq3s *w,
const struct janas_block_q8k *x, size_t nb);
/* Q4_1: as Q5_1 */
void janas_q4_1_dequantize(const struct janas_block_q4_1 *x, float *y,
size_t n);
float janas_q4_1_dot(const struct janas_block_q4_1 *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q4_1_dot_ref(const struct janas_block_q4_1 *w,
const struct janas_block_q8k *x, size_t nb);
/* IQ2_XXS, IQ2_XS, IQ2_S: one exact integer sum per 256 weights, scales
included, then one fma, as IQ3_S; w is nb blocks of the type */
void janas_iq2_dequantize(int type, const void *x, float *y, size_t n);
float janas_iq2_dot(int type, const void *w, const struct janas_block_q8k *x,
size_t nb);
float janas_iq2_dot_ref(int type, const void *w,
const struct janas_block_q8k *x, size_t nb);
/* Q2_K: Q4_K's scheme, an exact sum for the scales and one for the mins,
then an fma each */
void janas_q2k_dequantize(const struct janas_block_q2k *x, float *y, size_t n);
float janas_q2k_dot(const struct janas_block_q2k *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q2k_dot_ref(const struct janas_block_q2k *w,
const struct janas_block_q8k *x, size_t nb);
/* IQ1_M: (8 level + shift sign) times the half's 2 s + 1, one exact integer
sum per 256 weights, then one fma with d / 8 */
void janas_iq1m_dequantize(const struct janas_block_iq1m *x, float *y,
size_t n);
void janas_iq1s_dequantize(const struct janas_block_iq1s *x, float *y,
size_t n);
float janas_iq1s_dot(const struct janas_block_iq1s *w,
const struct janas_block_q8k *x, size_t nb);
float janas_iq1s_dot_ref(const struct janas_block_iq1s *w,
const struct janas_block_q8k *x, size_t nb);
float janas_iq1m_dot(const struct janas_block_iq1m *w,
const struct janas_block_q8k *x, size_t nb);
float janas_iq1m_dot_ref(const struct janas_block_iq1m *w,
const struct janas_block_q8k *x, size_t nb);
/* IQ3_XXS: one exact integer sum per 256 weights, scales included, then
one fma, as IQ3_S */
void janas_iq3xxs_dequantize(const struct janas_block_iq3xxs *x, float *y,
size_t n);
float janas_iq3xxs_dot(const struct janas_block_iq3xxs *w,
const struct janas_block_q8k *x, size_t nb);
float janas_iq3xxs_dot_ref(const struct janas_block_iq3xxs *w,
const struct janas_block_q8k *x, size_t nb);
/* Q5_0 (nb groups of eight blocks): per block position the exact sum of
(q - 16) x a in a lane of its own, the eight added as janas_hsum8 does */
void janas_q5_0_dequantize(const struct janas_block_q5_0 *x, float *y,
size_t n);
float janas_q5_0_dot(const struct janas_block_q5_0 *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q5_0_dot_ref(const struct janas_block_q5_0 *w,
const struct janas_block_q8k *x, size_t nb);
/* Q2_0 (nb groups of four blocks): a lane for each block of 32
activations, the exact sum of (q - 1) x a times d and their scale, the
eight added as janas_hsum8 does */
void janas_q2_0_dequantize(const struct janas_block_q2_0 *x, float *y,
size_t n);
float janas_q2_0_dot(const struct janas_block_q2_0 *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q2_0_dot_ref(const struct janas_block_q2_0 *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q8_0_dot(const struct janas_block_q8_0 *w,
const struct janas_block_q8k *x, size_t nb);
float janas_q8_0_dot_ref(const struct janas_block_q8_0 *w,
const struct janas_block_q8k *x, size_t nb);
#endif