37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* quant_q6k.c - Q6_K made from floats, and a row of Q6_K in bit planes added
* to a vector at the depth asked: what Janas's down matrix by neuron
* (JANAS_Q6_K_PN, tools/jns_neurons.c) is made of and read with.
*/
#include "quant.h"
#include "quant_q6kp.h"
#include <math.h>
#include <stdlib.h>
#include <string.h>
#if defined(__x86_64__)
#include <immintrin.h>
#endif
/* The integer of x at scale a, within Q6_K's -32 .. 31. */
static int q6(float x, float a)
{
if (a == 0.0f)
return 0;
int q = (int)lrintf(x / a);
return q < -32 ? -32 : q > 31 ? 31 : q;
}
/* The scale of a group of 16, among a few near its largest magnitude, of
least squared error. */
static float group_scale(const float *x)
{
float amax = 0.0f, ext = 0.0f;
for (int i = 0; i < 16; i++)
if (fabsf(x[i]) > amax) {
amax = fabsf(x[i]);
ext = x[i];
}
if (amax == 0.0f)
return 0.0f;
/* the largest one at the end of the range on its side: 31 up, 32 down */
float a0 = ext > 0 ? amax / 31.0f : amax / 32.0f, best = a0;
double best_err = -1;
for (int k = -6; k <= 6; k++) {
float a = a0 * (1.0f + 0.02f * (float)k);
double err = 0;
for (int i = 0; i < 16; i++) {
double e = x[i] - a * (float)q6(x[i], a);
err += e * e;
}
if (best_err < 0 || err < best_err) {
best_err = err;
best = a;
}
}
return best;
}
void janas_q6k_quantize(const float *x, struct janas_block_q6k *y, size_t n)
{
for (size_t b = 0; b < n / JANAS_QK; b++, x += JANAS_QK) {
float a[16], amax = 0.0f;
for (int s = 0; s < 16; s++) {
a[s] = group_scale(x + 16 * s);
if (a[s] > amax)
amax = a[s];
}
uint16_t d16 = janas_fp32_to_fp16(amax / 127.0f);
float d = janas_fp16_to_fp32(d16);
int8_t sc[16];
int q[JANAS_QK];
for (int s = 0; s < 16; s++) {
int v = d > 0 ? (int)lrintf(a[s] / d) : 0;
sc[s] = (int8_t)(v > 127 ? 127 : v < 0 ? 0 : v);
}
/* the values under the factors, then the block's factor by least
squares, x ~ d (sc q), twice */
for (int it = 0; it < 3; it++) {
for (int v = 0; v < JANAS_QK; v++)
q[v] = q6(x[v], d * sc[v / 16]);
if (it == 2)
break;
double sxu = 0, suu = 0;
for (int v = 0; v < JANAS_QK; v++) {
double u = (double)sc[v / 16] * q[v];
sxu += x[v] * u;
suu += u * u;
}
if (suu > 0 && sxu > 0) {
d16 = janas_fp32_to_fp16((float)(sxu / suu));
d = janas_fp16_to_fp32(d16);
}
}
memset(y[b].ql, 0, sizeof(y[b].ql));
memset(y[b].qh, 0, sizeof(y[b].qh));
memcpy(y[b].scales, sc, sizeof(sc));
y[b].d = d16;
/* the bits where q6k_value (quant.c) reads them */
for (int v = 0; v < JANAS_QK; v++) {
int u = q[v] + 32, half = v / 128, part = (v % 128) / 32,
l = v % 32;
uint8_t *ql = y[b].ql + 64 * half + (part & 1) * 32 + l;
*ql |= (uint8_t)((u & 0xf) << (part < 2 ? 0 : 4));
y[b].qh[32 * half + l] |= (uint8_t)((u >> 4) << (2 * part));
}
}
}
/* One block of the row: y += hd times each weight's value, on the scale
whose 64 is zero, times its group's factor - fused, as the vector one. */
static void axpy_block_ref(const struct janas_block_q6kp *b, const uint8_t *a1,
const uint8_t *a0, int level, float hd, float *y)
{
for (int v = 0; v < JANAS_QK; v++)
y[v] = fmaf(hd * (float)b->scales[v / 16],
(float)(q6kp_value2(b, a1, a0, v, level) - 64), y[v]);
}
#if defined(__x86_64__)
static int use_avx2, use_pn, use_vnni; /* use_pn: AVX2 and F16C */
/* Resolved before main, as quant.c's kernels; JANAS_KERNELS=avx2 leaves
AVX-VNNI out here too. */
__attribute__((constructor)) static void q6k_init(void)
{
__builtin_cpu_init();
use_avx2 = __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma");
const char *k = getenv("JANAS_KERNELS");
use_pn = use_avx2 && __builtin_cpu_supports("f16c");
use_vnni = use_pn && __builtin_cpu_supports("avxvnni") &&
!(k && !strcmp(k, "avx2"));
}
/* The same, 32 weights at a time: a byte of a plane holds the two bits of
four weights 32 apart (q6kp_index), so a part's shift and mask of the
planes' 32 bytes give 32 consecutive weights; their values, less 64, in
bytes that never carry; then floats, 8 at a time, times the factor of
their group of 16. Bit for bit axpy_block_ref. */
__attribute__((target("avx2,fma"))) static void
axpy_block_avx2(const struct janas_block_q6kp *b, const uint8_t *a1,
const uint8_t *a0, int level, float hd, float *y)
{
const __m256i m3 = _mm256_set1_epi8(3);
const __m256i off = _mm256_set1_epi8(level == 3 ? 64
: level == 2 ? 61
: 49);
const __m128i up = _mm_cvtsi32_si128(level == 3 ? 1 : level == 2 ? 3 : 5);
for (int n = 0; n < 2; n++) {
__m256i q = _mm256_loadu_si256((const __m256i *)(b->q + 32 * n));
__m256i p1 = level >= 2
? _mm256_loadu_si256((const __m256i *)(a1 + 32 * n))
: _mm256_setzero_si256();
__m256i p0 = level >= 3
? _mm256_loadu_si256((const __m256i *)(a0 + 32 * n))
: _mm256_setzero_si256();
for (int part = 0; part < 4; part++) {
__m128i sh = _mm_cvtsi32_si128(2 * part);
__m256i v = _mm256_and_si256(_mm256_srl_epi16(q, sh), m3);
if (level >= 2)
v = _mm256_or_si256(
_mm256_slli_epi16(v, 2),
_mm256_and_si256(_mm256_srl_epi16(p1, sh), m3));
if (level >= 3)
v = _mm256_or_si256(
_mm256_slli_epi16(v, 2),
_mm256_and_si256(_mm256_srl_epi16(p0, sh), m3));
/* 2v - 64, 8v + 3 - 64, 32v + 15 - 64 (q6kp_value2) */
v = _mm256_sub_epi8(_mm256_sll_epi16(v, up), off);
int v0 = 128 * n + 32 * part;
__m256 c0 = _mm256_set1_ps(hd * (float)b->scales[v0 / 16]);
__m256 c1 = _mm256_set1_ps(hd * (float)b->scales[v0 / 16 + 1]);
__m128i lo = _mm256_castsi256_si128(v),
hi = _mm256_extracti128_si256(v, 1);
__m256 w0 = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(lo));
__m256 w1 =
_mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(lo, 8)));
__m256 w2 = _mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(hi));
__m256 w3 =
_mm256_cvtepi32_ps(_mm256_cvtepi8_epi32(_mm_srli_si128(hi, 8)));
float *o = y + v0;
_mm256_storeu_ps(o, _mm256_fmadd_ps(c0, w0, _mm256_loadu_ps(o)));
_mm256_storeu_ps(o + 8,
_mm256_fmadd_ps(c0, w1, _mm256_loadu_ps(o + 8)));
_mm256_storeu_ps(o + 16,
_mm256_fmadd_ps(c1, w2, _mm256_loadu_ps(o + 16)));
_mm256_storeu_ps(o + 24,
_mm256_fmadd_ps(c1, w3, _mm256_loadu_ps(o + 24)));
}
}
}
#endif
void janas_q6kpn_axpy(const struct janas_block_q6kp *base, const uint8_t *p1,
const uint8_t *p0, int level, float h, float *y, size_t n)
{
if (level < 1)
return;
for (size_t b = 0; b < n / JANAS_QK; b++, y += JANAS_QK) {
const uint8_t *a1 = level >= 2 ? p1 + b * JANAS_Q6KP_PLANE : NULL;
const uint8_t *a0 = level >= 3 ? p0 + b * JANAS_Q6KP_PLANE : NULL;
float hd = h * (janas_fp16_to_fp32(base[b].d) * 0.5f);
#if defined(__x86_64__)
if (use_avx2) {
axpy_block_avx2(&base[b], a1, a0, level, hd, y);
continue;
}
#endif
axpy_block_ref(&base[b], a1, a0, level, hd, y);
}
}
void janas_q6kpn_axpy_ref(const struct janas_block_q6kp *base,
const uint8_t *p1, const uint8_t *p0, int level,
float h, float *y, size_t n)
{
if (level < 1)
return;
for (size_t b = 0; b < n / JANAS_QK; b++, y += JANAS_QK)
axpy_block_ref(&base[b], level >= 2 ? p1 + b * JANAS_Q6KP_PLANE : NULL,
level >= 3 ? p0 + b * JANAS_Q6KP_PLANE : NULL, level,
h * (janas_fp16_to_fp32(base[b].d) * 0.5f), y);
}
void janas_q6kpn_chunk_ref(const struct janas_block_q6kp *base,
const uint8_t *p1, const uint8_t *p0, size_t rows,
const uint32_t *js, const uint8_t *lv,
const float *h, uint32_t n, size_t b, float *y)
{
float u[JANAS_PN_CHUNK], t = 0.0f;
if (n > JANAS_PN_CHUNK)
n = JANAS_PN_CHUNK;
/* the factors' common scale: the largest |h d / 2| times 128, the
largest magnitude of an eight-bit scale */
for (uint32_t i = 0; i < n; i++) {
const struct janas_block_q6kp *w = base + b * rows + js[i];
u[i] = h[i] * (janas_fp16_to_fp32(w->d) * 0.5f);
if (fabsf(u[i]) > t)
t = fabsf(u[i]);
}
if (!(t > 0.0f))
return;
float inv = 127.0f / (t * 128.0f), f = t * 128.0f / 127.0f;
int8_t a[JANAS_PN_CHUNK][JANAS_QK / 16];
int32_t sum[JANAS_QK / 16] = {0};
for (uint32_t i = 0; i < n; i++) {
const struct janas_block_q6kp *w = base + b * rows + js[i];
float k = u[i] * inv;
for (int g = 0; g < JANAS_QK / 16; g++) { /* within -127 .. 127 */
long q = lrintf(k * (float)w->scales[g]);
a[i][g] = (int8_t)q;
sum[g] += (int32_t)q;
}
}
/* each output: the integer sum over the neurons, less 64 times the
factors' (the values' zero), times the scale */
for (int v = 0; v < JANAS_QK; v++) {
int32_t p = 0;
for (uint32_t i = 0; i < n; i++) {
size_t at = b * rows + js[i];
const uint8_t *x1 = lv[i] >= 2 ? p1 + at * JANAS_Q6KP_PLANE : NULL;
const uint8_t *x0 = lv[i] >= 3 ? p0 + at * JANAS_Q6KP_PLANE : NULL;
p += a[i][v / 16] * q6kp_value2(base + at, x1, x0, v, lv[i]);
}
y[v] = fmaf(f, (float)(p - 64 * sum[v / 16]), y[v]);
}
}
#if defined(__x86_64__)
/* The kernel proper: quant_pn.h, once for AVX2 and once for AVX-VNNI. */
#define PN(name) name##_avx2
#define PN_TARGET __attribute__((target("avx2,fma,f16c")))
#define PN_DOT(acc, u, s) \
_mm256_add_epi32(acc, _mm256_madd_epi16(_mm256_maddubs_epi16(u, s), \
_mm256_set1_epi16(1)))
#include "quant_pn.h"
#undef PN
#undef PN_TARGET
#undef PN_DOT
#define PN(name) name##_vnni
#define PN_TARGET __attribute__((target("avx2,fma,f16c,avxvnni")))
#define PN_DOT(acc, u, s) _mm256_dpbusd_avx_epi32(acc, u, s)
#include "quant_pn.h"
#undef PN
#undef PN_TARGET
#undef PN_DOT
#endif
void janas_q6kpn_chunk(const struct janas_block_q6kp *base, const uint8_t *p1,
const uint8_t *p0, size_t rows, const uint32_t *js,
const uint8_t *lv, const float *h, uint32_t n, size_t b,
float *y)
{
#if defined(__x86_64__)
if (use_vnni) {
q6kpn_chunk_vnni(base, p1, p0, rows, js, lv, h, n, b, y);
return;
}
if (use_pn) {
q6kpn_chunk_avx2(base, p1, p0, rows, js, lv, h, n, b, y);
return;
}
#endif
janas_q6kpn_chunk_ref(base, p1, p0, rows, js, lv, h, n, b, y);
}