37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* attention.h - causal grouped-query attention over an eight-bit KV cache
* (spans that attend both ways set call by call).
*
* The keys and values of one layer are laid out n_head_kv x n_ctx x
* head_dim as signed bytes, with one float scale per position and KV head
* (n_head_kv x n_ctx): the value of a weight is its byte times the scale of
* its position. Half the bytes of half precision, which at a long context is
* what attention costs, and the error is well under what the model's own
* quantization already carries. The positions of a KV head are contiguous. A
* call attends a block of n tokens at positions pos0 .. pos0 + n - 1, whose
* keys and values must already be in the cache. The result of a token does not
* depend on the block it runs in: blocks and token-by-token calls agree bit for
* bit.
*/
#ifndef JANAS_LLM_ATTENTION_H
#define JANAS_LLM_ATTENTION_H
#include <stddef.h>
#include <stdint.h>
#include "common/pool.h"
#define JANAS_ATTN_MAX_BLOCK 64
/*
* How the scores are computed. FLOAT reads the query as it is and sums in
* floating point. INT16 reads it as sixteen-bit integers with a scale of its
* own per token and head: the dot product is then an exact integer sum, which
* the machine does sixteen products at a time instead of eight - about twice
* as fast - at the cost of the query's quantization, a relative 3e-5; and it
* weighs the values with sixteen-bit integers too, a chunk's weights with one
* scale (attention.c, att_wquant_ref): a few 1e-4 on an output at most. AUTO
* takes INT16 when the instructions for it are there and the caller asked
* for a context longer than the usual one (more than 16384 positions), where
* attention is the greater part of a token; FLOAT otherwise, since below that
* the exact query costs little and a conversation may never reach the far
* end of the context anyway. The environment variable JANAS_ATTN (float,
* int16, auto) overrides the lot.
*/
enum janas_attn_scores {
JANAS_ATTN_AUTO = 0,
JANAS_ATTN_FLOAT = 1,
JANAS_ATTN_INT16 = 2
};
struct janas_attn;
/*
* The bytes of the partial results an instance keeps for a context of n_ctx
* positions: a softmax partial per chunk of positions, token of a block and
* query head. They grow with the context like the KV cache itself - 1.08 GB
* for Qwen3-Next-80B at 262,144 positions - so whoever sizes the memory has
* to count them (janas_llm_open does).
*/
size_t janas_attn_part_bytes(uint32_t n_head, uint32_t n_head_kv,
uint32_t head_dim, uint32_t n_ctx);
/* Scratch for a pool of n_threads threads; NULL when out of memory. */
struct janas_attn *janas_attn_create(uint32_t n_head, uint32_t n_head_kv,
uint32_t head_dim, uint32_t n_ctx,
int n_threads, int scores);
void janas_attn_destroy(struct janas_attn *a);
/* The scale of the scores, 1 / sqrt(head_dim) until set (Gemma 4: 1, its
q and k norms do the scaling). */
void janas_attn_set_scale(struct janas_attn *a, float scale);
/* A sliding window: token p sees positions p - window + 1 .. p only; 0 (the
default) for all. The cache still holds every position. */
void janas_attn_set_window(struct janas_attn *a, uint32_t window);
/* Keys and values of sixteen bits (int16 with the same scale per position
and KV head) instead of eight; the scores are then computed over the float
query (FLOAT). Twice the bytes, some 250 times finer: Gemma 4 needs it. */
void janas_attn_set_kv16(struct janas_attn *a, int on);
/*
* A ring (with a window): the cache holds n_ctx positions, n_ctx a multiple
* of 256, position p in row p mod n_ctx; a call must fit in it, window and
* the call's tokens together with room for the chunk boundaries
* (janas_attn_ring_positions). The partial results are kept for that many
* positions only.
*/
void janas_attn_set_ring(struct janas_attn *a, int on);
/* The positions a ring needs for a window and calls of up to n tokens
whose keys and values are written before any of them is attended. */
uint32_t janas_attn_ring_positions(uint32_t window, uint32_t n);
/*
* For the next call only: token j sees the positions up to last[j] instead
* of up to its own (pos0 + j), last[j] >= pos0 + j and never less than
* last[j - 1] - the tokens of a span that attends both ways, a picture's
* in Gemma 4 12B and 26B; the keys and values up to the largest must be in
* the cache. A window still starts from the token's own position. NULL:
* causal, as every call without it.
*/
void janas_attn_set_last(struct janas_attn *a, const uint32_t *last);
/* What the scores are actually computed with: FLOAT or INT16. */
int janas_attn_scores_mode(const struct janas_attn *a);
/* 1 when the engine chose it rather than the caller. */
int janas_attn_scores_auto(const struct janas_attn *a);
/*
* q: n x n_head x head_dim, token after token; k and v: the layer's cache,
* ks and vs their scales; out: n x n_head x head_dim. Scores are scaled by
* 1 / sqrt(head_dim) unless set otherwise.
*/
void janas_attn_run(struct janas_attn *a, struct janas_pool *pool,
const float *q, const int8_t *k, const float *ks,
const int8_t *v, const float *vs, uint32_t n, uint32_t pos0,
float *out);
#endif