37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* model_qsa.c - qwen4exp's sparse attention: past ix_top + ix_ratio - 1
* positions a query attends only to the ix_top / ix_ratio blocks of
* ix_ratio positions its indexer scores highest, and to the positions of
* its own block while that is incomplete.
*
* The indexer, in each attention layer: a key of ix_dim numbers per position
* (index_k_proj of the layer's input), kept raw; a block's ix_ratio keys
* pooled into one, their mean normalized (index_k_norm) and turned for the
* block's first position; a query per head (index_q_proj, normalized and
* turned for its own position); a block's score the sum over the heads of
* relu(q_h . key). The tail - positions (q + 1) / ratio * ratio to q, the
* query's own block until it is complete - is always attended. With no more
* complete blocks than ix_top / ix_ratio every position is, and the
* attention is the dense one: only past that is a query's row worked out
* again here, over the positions chosen.
*
* llama.cpp's build_qsa_top_k (src/models/qwen4exp.cpp) takes its top_k +
* ratio - 1 positions by score, so where the tail is shorter than ratio - 1
* it adds a few positions of one more block, chosen among equal scores by
* its sort; the reference it follows, and this file, keep whole blocks.
*/
#define _GNU_SOURCE
#include <math.h>
#include <stdlib.h>
#include <string.h>
#include "common/aligned.h"
#include "common/pool.h"
#include "jmath.h"
#include "vmath.h"
#include "model_impl.h"
int janas_m_qsa_load(struct janas_llm_model *m, char *err, size_t err_len)
{
char key[128];
uint32_t heads = 0, dim = 0, top = 0;
snprintf(key, sizeof(key), "%.32s.attention.indexer.head_count",
m->j.h.arch);
if (janas_jns_meta_u32(&m->j, key, &heads) != 0)
return 0; /* no indexer: dense attention throughout */
snprintf(key, sizeof(key), "%.32s.attention.indexer.key_length",
m->j.h.arch);
janas_jns_meta_u32(&m->j, key, &dim);
snprintf(key, sizeof(key), "%.32s.attention.indexer.top_k", m->j.h.arch);
janas_jns_meta_u32(&m->j, key, &top);
/* one ratio for every attention layer that has one: llama.cpp's input
set is per ratio, and Flash-Next's are all 4 */
uint32_t ratio = 0;
snprintf(key, sizeof(key), "%.32s.attention.compress_ratios", m->j.h.arch);
for (uint32_t l = 0; l < m->n_layer; l++) {
uint64_t r = 0;
if (janas_jns_meta_uint_at(&m->j, key, l, &r, NULL) != 0)
break;
if (m->layers[l].rec || r == 0)
continue;
if (ratio && r != ratio) {
snprintf(err, err_len, "sparse attention: ratios differ");
return -1;
}
ratio = (uint32_t)r;
}
if (!ratio)
return 0;
if (heads == 0 || heads > 64 || dim == 0 || dim % 8 || dim > m->head_dim ||
top == 0 || top % ratio || ratio > 64 || m->n_rot > dim ||
top + ratio > JANAS_QSA_MAX) {
snprintf(err, err_len, "sparse attention: inconsistent sizes");
return -1;
}
m->ix_heads = heads;
m->ix_dim = dim;
m->ix_top = top;
m->ix_ratio = ratio;
uint32_t dm = m->d_model;
char name[64];
for (uint32_t l = 0; l < m->n_layer; l++) {
struct layer *ly = &m->layers[l];
if (ly->rec)
continue;
static const char *const nm[4] = {"q_proj", "k_proj", "q_norm",
"k_norm"};
const struct janas_jns_tensor *t[4];
for (int i = 0; i < 4; i++) {
snprintf(name, sizeof(name), "blk.%u.indexer.%s.weight", l, nm[i]);
t[i] = janas_jns_tensor(&m->j, name);
uint64_t d0 = i < 2 ? dm : dim, d1 = i == 0 ? heads * dim : dim;
int ok = t[i] && t[i]->dims[0] == d0 &&
(i < 2 ? t[i]->dims[1] == d1 && t[i]->type == 30
: t[i]->n_dims == 1 && t[i]->type == 0);
if (!ok) {
snprintf(err, err_len, "tensor %s missing or unexpected", name);
return -1;
}
}
ly->ix_q = t[0];
ly->ix_k = t[1];
ly->ix_qn = t[2];
ly->ix_kn = t[3];
}
/* the raw keys of every position, at half precision as llama.cpp keeps
them, the pooled keys of every block, a block's queries and keys */
size_t B = m->blk, ctx = m->n_ctx, nbk = (ctx + ratio - 1) / ratio;
/* on whole pages: the GPU's whole token imports them (model_gpu.c) */
m->ix_raw = janas_aligned_alloc(
4096, ((size_t)m->n_attn * ctx * dim * sizeof(uint16_t) + 4095) &
~(size_t)4095);
m->ix_pool = janas_aligned_alloc(
4096,
((size_t)m->n_attn * nbk * dim * sizeof(float) + 4095) & ~(size_t)4095);
/* the bf16 products' (janas_gpu_dense_mm: a prompt's blocks) */
m->ix_qs = janas_m_out_alloc(m, B * heads * dim * sizeof(float));
m->ix_kb = janas_m_out_alloc(m, B * dim * sizeof(float));
m->ix_xb = janas_m_out_alloc(m, B * dm * sizeof(float));
m->ix_score = malloc(nbk * sizeof(float));
m->ix_sel = malloc((nbk + 1) * sizeof(uint32_t));
m->ix_pos = malloc(((size_t)top + ratio) * sizeof(uint32_t));
if (!m->ix_raw || !m->ix_pool || !m->ix_qs || !m->ix_kb || !m->ix_xb ||
!m->ix_score || !m->ix_sel || !m->ix_pos) {
snprintf(err, err_len, "out of memory for the sparse attention");
return -1;
}
return 0;
}
void janas_m_qsa_free(struct janas_llm_model *m)
{
janas_aligned_free(m->ix_raw);
janas_aligned_free(m->ix_pool);
janas_aligned_free(m->ix_qs);
janas_aligned_free(m->ix_kb);
janas_aligned_free(m->ix_xb);
free(m->ix_score);
free(m->ix_sel);
free(m->ix_pos);
}
/* NeoX rotation of the first n_rot numbers of v for position p, as the
attention turns its heads (model.c rope_table, model_attn.c) */
static void rope_at(const struct janas_llm_model *m, float *v, uint32_t p)
{
uint32_t half = m->n_rot / 2;
for (uint32_t i = 0; i < half; i++) {
float s, c,
theta = (float)p * janas_powf(m->rope_base,
-2.0f * (float)i / (float)m->n_rot);
janas_sincosf(theta, &s, &c);
float a = v[i], b = v[i + half];
v[i] = a * c - b * s;
v[i + half] = a * s + b * c;
}
}
void janas_m_qsa_rope(const struct janas_llm_model *m, float *out,
uint32_t pos0, uint32_t n)
{
uint32_t half = m->n_rot / 2, r = m->ix_ratio;
for (uint32_t j = 0; j < n + n / r + 2; j++) {
uint32_t p = j < n ? pos0 + j : (pos0 / r + (j - n)) * r;
float *row = out + (size_t)j * m->n_rot;
for (uint32_t i = 0; i < half; i++) {
float theta =
(float)p *
janas_powf(m->rope_base, -2.0f * (float)i / (float)m->n_rot);
janas_sincosf(theta, &row[half + i], &row[i]);
}
}
}
/* block b of slot sl pooled: the mean of its raw keys, normalized, turned
for its first position */
static void pool_block(struct janas_llm_model *m, const struct layer *ly,
uint32_t b)
{
uint32_t d = m->ix_dim, r = m->ix_ratio;
size_t nbk = (m->n_ctx + r - 1) / r;
const uint16_t *raw =
m->ix_raw + ((size_t)ly->slot * m->n_ctx + (size_t)b * r) * d;
float *out = m->ix_pool + ((size_t)ly->slot * nbk + b) * d;
for (uint32_t i = 0; i < d; i++) {
float s = 0.0f;
for (uint32_t k = 0; k < r; k++)
s += janas_fp16_to_fp32(raw[(size_t)k * d + i]);
out[i] = s / (float)r;
}
rms_norm(out, out, f32(m, ly->ix_kn), d, m->eps);
rope_at(m, out, b * r);
}
void janas_m_qsa_restore(struct janas_llm_model *m, uint32_t n)
{
for (uint32_t l = 0; m->ix_heads && l < m->n_layer; l++)
if (m->layers[l].ix_q)
for (uint32_t b = 0; (b + 1) * m->ix_ratio <= n; b++)
pool_block(m, &m->layers[l], b);
}
/* The context moved along: the raw keys after keep + drop come down drop
places (they carry no RoPE), and every block from keep's on is pooled
again, at its new place */
void janas_m_qsa_shift(struct janas_llm_model *m, uint32_t keep, uint32_t drop,
uint32_t n_kv)
{
if (!m->ix_heads)
return;
uint32_t d = m->ix_dim, r = m->ix_ratio, n = n_kv - drop;
for (uint32_t l = 0; l < m->n_layer; l++) {
const struct layer *ly = &m->layers[l];
if (!ly->ix_q)
continue;
uint16_t *raw = m->ix_raw + (size_t)ly->slot * m->n_ctx * d;
memmove(raw + (size_t)keep * d, raw + (size_t)(keep + drop) * d,
(size_t)(n - keep) * d * sizeof(uint16_t));
for (uint32_t b = keep / r; (b + 1) * r <= n; b++)
pool_block(m, ly, b);
}
}
struct score_job {
struct janas_llm_model *m;
const float *pool, *q; /* the layer's pooled keys, the query's heads */
uint32_t n_blocks;
};
static void score_worker(void *arg, int tid, int n_threads)
{
struct score_job *s = arg;
struct janas_llm_model *m = s->m;
uint32_t d = m->ix_dim;
uint32_t per =
(s->n_blocks + (uint32_t)n_threads - 1) / (uint32_t)n_threads;
uint32_t a = (uint32_t)tid * per,
e = a + per < s->n_blocks ? a + per : s->n_blocks;
for (uint32_t b = a; b < e; b++) {
const float *k = s->pool + (size_t)b * d;
float sum = 0.0f;
for (uint32_t h = 0; h < m->ix_heads; h++) {
float dot = janas_dot_f32(s->q + (size_t)h * d, k, d);
sum += dot > 0.0f ? dot : 0.0f;
}
m->ix_score[b] = sum;
}
}
/* The k-th largest of v[0 .. n) (k < n), v reordered: Hoare's selection */
static float kth_largest(float *v, uint32_t n, uint32_t k)
{
uint32_t lo = 0, hi = n - 1;
while (lo < hi) {
float pivot = v[lo + (hi - lo) / 2];
uint32_t i = lo, j = hi;
while (i <= j) {
while (v[i] > pivot)
i++;
while (v[j] < pivot)
j--;
if (i <= j) {
float t = v[i];
v[i] = v[j];
v[j] = t;
i++;
if (j == 0)
break;
j--;
}
}
if (k <= j)
hi = j;
else if (k >= i)
lo = i;
else
break;
}
return v[k];
}
/* The keep blocks of highest score among blocks 0 .. n - 1 (n > keep), in
their order into sel: those above the keep-th score, then among those
equal to it the earliest */
static void choose(struct janas_llm_model *m, uint32_t n, uint32_t keep)
{
float *tmp = (float *)(void *)m->ix_sel; /* as many floats as blocks */
memcpy(tmp, m->ix_score, n * sizeof(float));
float th = kth_largest(tmp, n, keep - 1);
uint32_t above = 0;
for (uint32_t b = 0; b < n; b++)
above += m->ix_score[b] > th;
uint32_t equal = keep - above, k = 0;
for (uint32_t b = 0; b < n && k < keep; b++)
if (m->ix_score[b] > th || (m->ix_score[b] == th && equal && equal--))
m->ix_pos[k++] = b;
}
struct head_job {
struct janas_llm_model *m;
const struct layer *ly;
const float *q; /* the token's query heads, after norm and RoPE */
float *out; /* its attention row */
const uint32_t *pos;
uint32_t n_pos;
};
/* one query head over the chosen positions */
static void head_worker(void *arg, int tid, int n_threads)
{
struct head_job *t = arg;
struct janas_llm_model *m = t->m;
const struct layer *ly = t->ly;
uint32_t hd = ly->hd, nkv = ly->n_kv, rep = m->n_head / nkv;
uint32_t cap = m->kv_cap[ly->slot];
float scale = 1.0f / sqrtf((float)hd);
float sc[JANAS_QSA_MAX], kf[512];
for (uint32_t h = (uint32_t)tid; h < m->n_head; h += (uint32_t)n_threads) {
uint32_t g = h / rep;
const float *q = t->q + (size_t)h * hd;
const float *ks = m->kscale + m->ks_off[ly->slot] + (size_t)g * cap;
const float *vs = m->vscale + m->ks_off[ly->slot] + (size_t)g * cap;
float mx = -INFINITY;
for (uint32_t i = 0; i < t->n_pos; i++) {
uint32_t p = t->pos[i];
size_t at = ((size_t)g * cap + p) * hd;
if (m->kv16) {
const int16_t *k =
(const int16_t *)(const void *)(m->kcache +
m->kv_off[ly->slot]) +
at;
for (uint32_t e = 0; e < hd; e++)
kf[e] = (float)k[e];
} else {
const int8_t *k = m->kcache + m->kv_off[ly->slot] + at;
for (uint32_t e = 0; e < hd; e++)
kf[e] = (float)k[e];
}
sc[i] = janas_dot_f32(q, kf, hd) * ks[p] * scale;
if (sc[i] > mx)
mx = sc[i];
}
double sum = 0.0;
for (uint32_t i = 0; i < t->n_pos; i++) {
sc[i] = janas_expf(sc[i] - mx);
sum += sc[i];
}
float *out = t->out + (size_t)h * hd;
memset(out, 0, hd * sizeof(float));
for (uint32_t i = 0; i < t->n_pos; i++) {
uint32_t p = t->pos[i];
size_t at = ((size_t)g * cap + p) * hd;
float w = (float)(sc[i] / sum) * vs[p];
if (m->kv16) {
const int16_t *v =
(const int16_t *)(const void *)(m->vcache +
m->kv_off[ly->slot]) +
at;
for (uint32_t e = 0; e < hd; e++)
out[e] += w * (float)v[e];
} else {
const int8_t *v = m->vcache + m->kv_off[ly->slot] + at;
for (uint32_t e = 0; e < hd; e++)
out[e] += w * (float)v[e];
}
}
}
}
void janas_m_qsa_keys(struct janas_llm_model *m, const struct layer *ly,
const float *xb, uint32_t n, uint32_t pos0)
{
if (!m->ix_heads || !ly->ix_q)
return;
uint32_t dm = m->d_model, d = m->ix_dim, r = m->ix_ratio;
janas_m_bf16_product(m, data(m, ly->ix_k), xb, m->ix_kb, d, dm, n, 1.0f);
uint16_t *raw = m->ix_raw + (size_t)ly->slot * m->n_ctx * d;
for (uint32_t j = 0; j < n; j++)
for (uint32_t i = 0; i < d; i++)
raw[(size_t)(pos0 + j) * d + i] =
janas_fp32_to_fp16(m->ix_kb[(size_t)j * d + i]);
for (uint32_t b = pos0 / r; (b + 1) * r <= pos0 + n; b++)
pool_block(m, ly, b);
}
void janas_m_qsa(struct janas_llm_model *m, const struct layer *ly, uint32_t n,
uint32_t pos0)
{
if (!m->ix_heads || !ly->ix_q)
return;
uint32_t dm = m->d_model, d = m->ix_dim, r = m->ix_ratio, nh = m->ix_heads;
size_t nbk = (m->n_ctx + r - 1) / r;
/* the indexer's keys and queries from the layer's input, at bf16 */
for (size_t i = 0; i < (size_t)n * dm; i++)
m->ix_xb[i] = janas_m_round_bf16(m->xn[i]);
janas_m_bf16_product(m, data(m, ly->ix_k), m->ix_xb, m->ix_kb, d, dm, n,
1.0f);
janas_m_bf16_product(m, data(m, ly->ix_q), m->ix_xb, m->ix_qs, nh * d, dm,
n, 1.0f);
uint16_t *raw = m->ix_raw + (size_t)ly->slot * m->n_ctx * d;
for (uint32_t j = 0; j < n; j++) {
for (uint32_t i = 0; i < d; i++)
raw[(size_t)(pos0 + j) * d + i] =
janas_fp32_to_fp16(m->ix_kb[(size_t)j * d + i]);
for (uint32_t h = 0; h < nh; h++) {
float *q = m->ix_qs + ((size_t)j * nh + h) * d;
rms_norm(q, q, f32(m, ly->ix_qn), d, m->eps);
rope_at(m, q, pos0 + j);
}
}
/* the blocks this call completed, or wrote again */
for (uint32_t b = pos0 / r; (b + 1) * r <= pos0 + n; b++)
pool_block(m, ly, b);
const float *pool = m->ix_pool + (size_t)ly->slot * nbk * d;
uint32_t keep = m->ix_top / r, qd = m->n_head * ly->hd;
for (uint32_t j = 0; j < n; j++) {
uint32_t p = pos0 + j, tail = (p + 1) / r * r, blocks = tail / r;
if (blocks <= keep)
continue; /* every position: the dense attention stands */
struct score_job sj = {.m = m,
.pool = pool,
.q = m->ix_qs + (size_t)j * nh * d,
.n_blocks = blocks};
janas_pool_run(m->compute, score_worker, &sj);
/* the blocks chosen, then their positions, in place from the end */
choose(m, blocks, keep);
uint32_t np = keep * r;
for (uint32_t i = keep; i-- > 0;)
for (uint32_t k = r; k-- > 0;)
m->ix_pos[i * r + k] = m->ix_pos[i] * r + k;
for (uint32_t t = tail; t <= p; t++)
m->ix_pos[np++] = t;
struct head_job hj = {.m = m,
.ly = ly,
.q = m->q + (size_t)j * qd,
.out = m->att + (size_t)j * qd,
.pos = m->ix_pos,
.n_pos = np};
janas_pool_run(m->compute, head_worker, &hj);
}
}