Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

5.3 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * attention.h - causal grouped-query attention over an eight-bit KV cache
 * (spans that attend both ways set call by call).
 *
 * The keys and values of one layer are laid out n_head_kv x n_ctx x
 * head_dim as signed bytes, with one float scale per position and KV head
 * (n_head_kv x n_ctx): the value of a weight is its byte times the scale of
 * its position. Half the bytes of half precision, which at a long context is
 * what attention costs, and the error is well under what the model's own
 * quantization already carries. The positions of a KV head are contiguous. A
 * call attends a block of n tokens at positions pos0 .. pos0 + n - 1, whose
 * keys and values must already be in the cache. The result of a token does not
 * depend on the block it runs in: blocks and token-by-token calls agree bit for
 * bit.
 */
#ifndef JANAS_LLM_ATTENTION_H
#define JANAS_LLM_ATTENTION_H

#include <stddef.h>
#include <stdint.h>

#include "common/pool.h"

#define JANAS_ATTN_MAX_BLOCK 64

/*
 * How the scores are computed. FLOAT reads the query as it is and sums in
 * floating point. INT16 reads it as sixteen-bit integers with a scale of its
 * own per token and head: the dot product is then an exact integer sum, which
 * the machine does sixteen products at a time instead of eight - about twice
 * as fast - at the cost of the query's quantization, a relative 3e-5; and it
 * weighs the values with sixteen-bit integers too, a chunk's weights with one
 * scale (attention.c, att_wquant_ref): a few 1e-4 on an output at most. AUTO
 * takes INT16 when the instructions for it are there and the caller asked
 * for a context longer than the usual one (more than 16384 positions), where
 * attention is the greater part of a token; FLOAT otherwise, since below that
 * the exact query costs little and a conversation may never reach the far
 * end of the context anyway. The environment variable JANAS_ATTN (float,
 * int16, auto) overrides the lot.
 */
enum janas_attn_scores {
    JANAS_ATTN_AUTO = 0,
    JANAS_ATTN_FLOAT = 1,
    JANAS_ATTN_INT16 = 2
};

struct janas_attn;

/*
 * The bytes of the partial results an instance keeps for a context of n_ctx
 * positions: a softmax partial per chunk of positions, token of a block and
 * query head. They grow with the context like the KV cache itself - 1.08 GB
 * for Qwen3-Next-80B at 262,144 positions - so whoever sizes the memory has
 * to count them (janas_llm_open does).
 */
size_t janas_attn_part_bytes(uint32_t n_head, uint32_t n_head_kv,
                             uint32_t head_dim, uint32_t n_ctx);

/* Scratch for a pool of n_threads threads; NULL when out of memory. */
struct janas_attn *janas_attn_create(uint32_t n_head, uint32_t n_head_kv,
                                     uint32_t head_dim, uint32_t n_ctx,
                                     int n_threads, int scores);
void janas_attn_destroy(struct janas_attn *a);

/* The scale of the scores, 1 / sqrt(head_dim) until set (Gemma 4: 1, its
   q and k norms do the scaling). */
void janas_attn_set_scale(struct janas_attn *a, float scale);
/* A sliding window: token p sees positions p - window + 1 .. p only; 0 (the
   default) for all. The cache still holds every position. */
void janas_attn_set_window(struct janas_attn *a, uint32_t window);
/* Keys and values of sixteen bits (int16 with the same scale per position
   and KV head) instead of eight; the scores are then computed over the float
   query (FLOAT). Twice the bytes, some 250 times finer: Gemma 4 needs it. */
void janas_attn_set_kv16(struct janas_attn *a, int on);
/*
 * A ring (with a window): the cache holds n_ctx positions, n_ctx a multiple
 * of 256, position p in row p mod n_ctx; a call must fit in it, window and
 * the call's tokens together with room for the chunk boundaries
 * (janas_attn_ring_positions). The partial results are kept for that many
 * positions only.
 */
void janas_attn_set_ring(struct janas_attn *a, int on);
/* The positions a ring needs for a window and calls of up to n tokens
   whose keys and values are written before any of them is attended. */
uint32_t janas_attn_ring_positions(uint32_t window, uint32_t n);

/*
 * For the next call only: token j sees the positions up to last[j] instead
 * of up to its own (pos0 + j), last[j] >= pos0 + j and never less than
 * last[j - 1] - the tokens of a span that attends both ways, a picture's
 * in Gemma 4 12B and 26B; the keys and values up to the largest must be in
 * the cache. A window still starts from the token's own position. NULL:
 * causal, as every call without it.
 */
void janas_attn_set_last(struct janas_attn *a, const uint32_t *last);

/* What the scores are actually computed with: FLOAT or INT16. */
int janas_attn_scores_mode(const struct janas_attn *a);
/* 1 when the engine chose it rather than the caller. */
int janas_attn_scores_auto(const struct janas_attn *a);

/*
 * q: n x n_head x head_dim, token after token; k and v: the layer's cache,
 * ks and vs their scales; out: n x n_head x head_dim. Scores are scaled by
 * 1 / sqrt(head_dim) unless set otherwise.
 */
void janas_attn_run(struct janas_attn *a, struct janas_pool *pool,
                    const float *q, const int8_t *k, const float *ks,
                    const int8_t *v, const float *vs, uint32_t n, uint32_t pos0,
                    float *out);

#endif