Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

13.2 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * gpu_token.h - a whole token on the GPU, with one submission: every layer's
 * norms, products, RoPE, KV cache and attention, the feed-forward and the
 * output head recorded in one command buffer, the logits back at the end
 * (private to the library).
 *
 * For now the dense attention models (qwen3's shape: q and k norms, NeoX
 * RoPE, an eight-bit KV cache, SiLU feed-forward), and qwen3.5's (Gated
 * DeltaNet layers between them, the query with an output gate) for blocks,
 * both with a mixture of experts too (qwen3moe, qwen35moe: a submission a
 * layer, the CPU sorting the router's choice in between),
 * on a GPU that shares the host's memory: the weights, the experts' arena and
 * the KV cache are the CPU's own memory, imported, so the CPU's passes (a
 * prompt in blocks) and the GPU's tokens read and write the same cache. The
 * products are the CPU's bit for bit (their shaders); the norms and the
 * attention sum in the GPU's order.
 */
#ifndef JANAS_LLM_GPU_TOKEN_H
#define JANAS_LLM_GPU_TOKEN_H

#include <stdint.h>

#include "gpu.h"
#include "matvec.h"

/* One layer: host pointers, all in regions the GPU holds. The products'
   tasks give type, weights (and planes), rows and cols; their x, y and
   n_vec are the GPU's own. */
struct janas_gpu_tok_layer {
    const float *attn_norm, *q_norm, *k_norm, *ffn_norm;
    struct janas_matvec_task wq, wk, wv, wo, gate, up, down;
    /* the attention's geometry, the model's or the layer's kind's (Gemma
       4: sliding-window layers and full ones): head size, KV heads, RoPE
       dimensions, the positions its cache holds (a ring of them where
       ring: position p in row p mod cap), the window a token sees (0: all
       of them); wv's type 0: V is K before its norm (Gemma 4's full
       layers). Gemma 4's sandwich norms of the attention's and the
       feed-forward's outputs (NULL: none) and the layer's output scale */
    uint32_t hd, n_kv, n_rot, cap, window;
    int ring, swa;
    const float *post_attn, *post_ffn;
    float out_scale;
    /* Gemma 4's per-layer embeddings (E2B, E4B; NULL: none): the branch's
       gate (ple_dim x dm) and projection (dm x ple_dim), f32, its norm; the
       layer's keys and values another's (n_kv_layers on: wk's type 0) */
    const float *ple_gate, *ple_proj, *ple_post;
    /* Gemma 4's mixture (26B; pre_norm2 NULL: none): its dense branch (the
       shared expert, weight 1, no gate: sh_gate_w's type 0), normalized by
       post_norm1; the experts' input by pre_norm2, the router's the
       residual without weights times rscale / sqrt(dm), each expert's
       output scale, the experts' sum normalized by post_norm2 */
    struct janas_matvec_task dn_gate, dn_up, dn_down;
    const float *pre_norm2, *post_norm1, *post_norm2, *rscale, *exp_scale;
    int8_t *kc, *vc; /* the layer's KV cache: n_kv x cap x hd bytes */
    float *ks, *vs;  /* its scales: n_kv x cap */
    /* a Gated DeltaNet layer (qwen3.5) in place of the attention: q|k|v and
       z products, beta then alpha (2 nv x dm, f32), the convolution
       (d_conv x conv_ch, transposed), dt and A (nv), the heads' norm (ds),
       the output product; its state slot (an attention layer's: its KV
       slot, its row of att_in) */
    int rec;
    struct janas_matvec_task ssm_qkv, ssm_z, ssm_out;
    const float *ssm_beta, *ssm_alpha, *conv_w, *dt, *ssm_a, *ssm_norm;
    uint32_t slot;
    /* a mixture's layer (the model's n_expert > 1): the router (n_expert x
       dm, f16) and the shared expert's gate (dm floats, or NULL); gate, up
       and down the experts' types and shapes, with no weights - those come
       from the model's slots (janas_gpu_tok_model fetch): off, each
       matrix's bytes into a slot, poff the down matrix's two bare planes'
       (Q6_K_P: 0, a plane the cache does not hold); sh_gate_w, sh_up, sh_down
       the shared expert (type 0: none) */
    const uint16_t *router16;
    const float *sh_gate;
    size_t off[3], poff[2];
    struct janas_matvec_task sh_gate_w, sh_up, sh_down;
    /* qwen4exp (the model's hc): its mixes of the streams in place of the
       norms, [0] before the token mixer and [1] before the feed-forward:
       each stream's norm (hc x dm), down (rank x hc dm), up (hc dm x rank)
       and inject (hc x hc dm), bf16; on its n-gram layer the key product
       (hc dm x dm), the value (dm x dm, bf16), the key's, query's and
       conv's norms (hc dm each) and the conv's taps (hc dm x kernel,
       channel-major) */
    const float *hc_norm[2];
    const uint16_t *hc_down[2], *hc_up[2], *hc_inject[2];
    struct janas_matvec_task ng_key;
    const uint16_t *ng_value;
    const float *ng_nk, *ng_nq, *ng_nc, *ng_conv;
    /* qwen4exp's indexer on an attention layer (the model's ix_heads; or
       NULL): its query and key products (ix_heads ix_dim x dm and ix_dim x
       dm, bf16), their norms (ix_dim), the raw keys of every position
       (cap x ix_dim, half precision) and the pooled keys of every block
       (cap / ix_ratio x ix_dim), the model's: the CPU reads and writes
       them too */
    const uint16_t *ix_q, *ix_k;
    const float *ix_qn, *ix_kn;
    uint16_t *ix_raw;
    float *ix_pool;
};

struct janas_gpu_tok_model {
    /* n_kv, hd and n_rot the largest of any layer's, cap the full
       layers' */
    uint32_t n_layer, dm, n_head, n_kv, hd, n_rot, ff, n_vocab, cap;
    float eps, attn_scale;
    /* Gemma 4: GELU in place of SiLU, keys and values at sixteen bits (q
       left in float for them), V normalized without weights; the
       sliding-window layers' RoPE dimensions and their tables (n_rot_swa
       / 2 cos and sin a token, set before each run), the most tokens a
       block the rings leave room for (0: no ring) */
    int gelu, kv16, vnorm;
    uint32_t n_rot_swa, ring_room;
    const float *cs_swa, *sn_swa;
    /* Gemma 4's per-layer embeddings: ple_dim numbers a layer and token
       (0: none), the block's from the CPU (janas_m_ple_inputs, n_layer
       ple_dim a token), set before each run */
    uint32_t ple_dim;
    const float *ple_in;
    const struct janas_gpu_tok_layer *layers;
    const float *out_norm;
    struct janas_matvec_task output;
    /* for blocks of more than one token, where layers and output hold the
       GPU's single-vector copies (JANAS_GPU_REPACK): the weights as they
       are; NULL: layers and output */
    const struct janas_gpu_tok_layer *blayers;
    struct janas_matvec_task boutput;
    /* qwen3.5: wq's rows head by head, each head's q then its output gate;
       the recurrent layers' shapes and their states, two halves each (the
       model's double buffer): n_rec x nv x ds x ds sixteen-bit numbers and
       n_rec x (d_conv - 1) x conv_ch floats */
    int gated_q;
    uint32_t n_rec, ds, nv, nkh, d_conv, conv_ch, d_inner;
    int vmap_mod;
    uint16_t *ssm_state;
    float *conv_state;
    /* their log (the model's, model_impl.h): per recurrent layer log_rows
       rows of conv_ch floats (the convolution's inputs) and of nv floats
       (betas, decays), what a call replays of the last one when that one's
       last tokens were taken back (rejected drafts); NULL: blocks short
       enough to be logged stay on the CPU */
    float *log_mixed, *log_beta, *log_decay;
    uint32_t log_rows;
    /* a mixture (n_expert > 1, 0 for a dense model): k experts a token, of
       eff rows each; ff above is the shared expert's (eff without one).
       Its layers run one at a time, the CPU between two of them: the
       router's choice read, its experts' slots asked of fetch (0, or -1:
       the block goes to the CPU), the pairs given to routed (or NULL) */
    uint32_t n_expert, k, eff;
    int (*fetch)(void *ctx, uint32_t layer, const uint32_t *ids, size_t n,
                 const uint8_t **slots);
    void (*routed)(void *ctx, uint32_t layer, uint32_t n, uint32_t k,
                   const uint32_t *ids);
    /* reads every expert of the layer that is not in the cache, while the
       caller goes on (or NULL): a block of many tokens chooses nearly all
       of them, so none is read for nothing */
    void (*prefetch)(void *ctx, uint32_t layer);
    /* whether routed must see each layer's choice (or NULL): then a layer
       at a time even when the card holds every expert */
    int (*watched)(void *ctx);
    /* a card whose memory does not hold all the experts (tier_bytes > 0):
       a tier of slots of slot_bytes in its memory keeps those used most,
       read into it from the slots fetch gives; the others computed by
       cpu_rows, layer's experts ids[0..nd) of slots[], their rows
       [ra[i], rb[i]) of out (dm floats each, before their weights), row r
       a pair of token tok[r], from xn (n tokens of dm floats, the
       feed-forward's normalized input): 0, or -1 (the block goes to the
       CPU) */
    size_t tier_bytes, slot_bytes;
    int (*cpu_rows)(void *ctx, uint32_t layer, uint32_t n, const float *xn,
                    uint32_t nd, const uint32_t *ids,
                    const uint8_t *const *slots, const uint32_t *ra,
                    const uint32_t *rb, const uint32_t *tok, float *out);
    void *ctx;
    /* qwen4exp: hc streams of the residual (0: one, the others), the
       mixes' rank, the output's mix (no inject); ng_layer the n-gram
       layer (n_layer: none), its conv's kernel and dilation, its ring of
       inputs (ng_cap positions of hc dm floats, the model's: the CPU's
       passes read and write it too) and the block's embeddings, which the
       caller writes before each run (n x dm); att_in each attention
       layer's input as the GPU made it (n_attn x the block's n x dm), for
       the CPU's indexer keys (model_qsa.c) after the run, or NULL. The
       feed-forwards' widths (eff, ff) may end inside a group of 256: then
       their buffers are padded to whole groups of zeros. */
    uint32_t hc, hc_rank;
    /* the mixes' down and up rewritten as Q8_0 (JANAS_HC_MIX=q8: blocks
       of 32, rows of a whole number of them), their injects still bf16 */
    int hc_q8;
    const float *hco_norm;
    const uint16_t *hco_down, *hco_up;
    uint32_t ng_layer, ng_kernel, ng_len, ng_cap;
    float *ng_hist;
    const float *ng_emb;
    float *att_in;
    /* qwen4exp's sparse attention (0 heads: none): the indexer's heads,
       their width, the positions a token keeps past the dense ones and the
       blocks' length; ix_rope the cos and sin (n_rot floats a row) of the
       block's n tokens' positions, then of the first position of each
       block from pos / ix_ratio on (n / ix_ratio + 2 rows), which the
       caller writes before each run as model_qsa.c turns them; it needs
       att_in */
    uint32_t ix_heads, ix_dim, ix_top, ix_ratio;
    const float *ix_rope;
};

struct janas_gpu_tok;

/* The most tokens a run takes (janas_gpu_tok_max_block's), and the rows a
   layer has in att_in. */
#define JANAS_GPU_TOK_BLOCK 256

/* The most tokens of the last call a block replays before its own (see
   janas_gpu_tok_run): a chain of drafts is shorter. */
#define JANAS_GPU_TOK_MAX_REPLAY 16

/* Buffers and shaders for a model of that shape; NULL (the GPU cannot, or
   out of memory) with the reason in err. */
struct janas_gpu_tok *janas_gpu_tok_create(struct janas_gpu *g,
                                           const struct janas_gpu_tok_model *md,
                                           char *err, size_t err_len);
void janas_gpu_tok_destroy(struct janas_gpu *g, struct janas_gpu_tok *t);

/* The most tokens janas_gpu_tok_run takes at once (0 without t). */
uint32_t janas_gpu_tok_max_block(const struct janas_gpu_tok *t);
/* ... starting at position pos: fewer as the context grows (the
   attention's runs take room). */
uint32_t janas_gpu_tok_block_at(const struct janas_gpu_tok *t,
                                const struct janas_gpu_tok_model *md,
                                uint32_t pos);

/*
 * A block of n tokens (1 .. janas_gpu_tok_max_block) whose embeddings are x
 * (n x dm floats), token j at position pos + j (its keys and values go to
 * row pos + j of the cache; positions 0 .. pos + j attended), cs and sn
 * their RoPE tables (n x n_rot / 2 each): the logits (n_vocab each) of
 * the last token (all 0), of every token (1) or of none (2) into logits;
 * with hidden, the final hidden states (n x dm: the last norm's output, an
 * MTP block's input) there. The recurrent layers read their state from
 * half src and write it to half dst (the same for a block's later pieces);
 * first they replay the first replay tokens (at most
 * JANAS_GPU_TOK_MAX_REPLAY) of the log on half src in place, as the CPU
 * does when a call starts before the end of the last one (src is then the
 * base); with log_row other than UINT32_MAX they log the block's tokens
 * from that row of the log on.
 * 0; -1 with nothing changed but the cache rows pos .. pos + n - 1 (the
 * caller runs the block on the CPU then); -2 the GPU failed after it had
 * started, with the state no longer what it was.
 */
int janas_gpu_tok_run(struct janas_gpu *g, struct janas_gpu_tok *t,
                      const struct janas_gpu_tok_model *md, const float *x,
                      const float *cs, const float *sn, uint32_t pos,
                      uint32_t n, int all, float *logits, float *hidden,
                      uint32_t src, uint32_t dst, uint32_t replay,
                      uint32_t log_row);

#endif