37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* gpu.h - an optional integrated GPU for quantized products (Vulkan).
*
* Vulkan is loaded at run time: without a driver or a suitable GPU,
* janas_gpu_create returns NULL and the engine runs on the CPU alone. The
* GPU reads the model's weights where they are (host memory imported, no
* copy) and computes with the same arithmetic as the CPU kernels, so a
* product split between the two gives the same bits as either alone.
*
* A product group is split by rows: janas_gpu_begin submits the first rows
* of each task to the GPU (one submission for the group), the caller
* computes the other rows on the CPU meanwhile, janas_gpu_finish waits and
* writes the GPU's rows into the tasks' outputs. One group at a time.
*/
#ifndef JANAS_LLM_GPU_H
#define JANAS_LLM_GPU_H
#include <stddef.h>
#include <stdint.h>
#include "matvec.h"
struct janas_gpu;
/*
* Q6_K in the GPU's aligned copy (JANAS_GPU_REPACK=1 with the whole token,
* model_gpu.c): each 210-byte block padded to 224 - ql 128, qh 64, scales
* 16, d at byte 208 - so that every part starts on 16 bytes. Only the GPU
* reads it, only for single vectors; a Q6_K_P block with its three planes
* becomes the Q6_K block it came from.
*/
#define JANAS_Q6_K_A 216
#define JANAS_Q6KA_BLOCK 224
/*
* The single token's copies in tiles of JANAS_TILE rows (JANAS_GPU_REPACK=2:
* the Q6_K and the Q4_K matrices with rows of 32 blocks and more; 3: every
* Q4_K), block b of a tile's rows side by side, so that the subgroups (a
* row each) read neighbouring memory at the same time: faster for long
* rows (Qwen3-4B's down matrices), slower for short ones on the
* development laptop's integrated GPU. Block b of row r is block ((r / T)
* nb + b) T + r % T; rows padded to a whole tile.
*/
#define JANAS_TILE 64
#define JANAS_Q4_K_T 217
#define JANAS_Q6_K_AT 218
struct janas_gpu *janas_gpu_create(char *err, size_t err_len);
void janas_gpu_destroy(struct janas_gpu *g);
/* Makes host memory readable by the GPU: p and bytes multiples of 4096.
Returns 0 or -1. */
int janas_gpu_import(struct janas_gpu *g, const void *p, size_t bytes);
/*
* Memory allocated by the GPU driver, mapped and cached for the host, that
* the GPU reads with no import: bytes rounded up to 4096, NULL on failure,
* released by janas_gpu_destroy. After writing it on the CPU, call
* janas_gpu_flush before the GPU reads it.
*/
void *janas_gpu_alloc(struct janas_gpu *g, size_t bytes);
void janas_gpu_flush(struct janas_gpu *g, const void *p, size_t bytes);
/* Whether the GPU can compute this task (type supported, weights imported,
sizes within its buffers). */
int janas_gpu_can(const struct janas_gpu *g, const struct janas_matvec_task *t);
/*
* Submits rows [0, gpu_rows[i]) of each task (tasks the GPU cannot compute
* must have gpu_rows[i] = 0). Returns 0, or -1 with nothing submitted.
*/
int janas_gpu_begin(struct janas_gpu *g, const struct janas_matvec_task *tasks,
const size_t *gpu_rows, size_t n);
/*
* Experts on the GPU, bit for bit as the CPU computes them: for each expert
* i, gate_i x and up_i x (ff rows), h = SiLU(gate) * up quantized to Q8_K
* (the deterministic functions of vmath.h and quant.c), then down_i h (d
* rows), stored at out + i * d by janas_gpu_finish. x is one Q8_K vector of
* d values. One submission; the weights must be in shared or copied
* regions. Returns 0, or -1 with nothing submitted.
*/
struct janas_gpu_expert {
const void *gate, *up, *down;
int gate_type, up_type, down_type; /* JANAS_Q4_K or JANAS_Q6_K */
};
int janas_gpu_ffn_begin(struct janas_gpu *g, const struct janas_gpu_expert *e,
size_t n, const struct janas_block_q8k *x, size_t d,
size_t ff, float *out);
/* Waits for the submitted rows and stores them. Returns 0 or -1. Sets
*waited to whether the GPU was still running when called. */
int janas_gpu_finish(struct janas_gpu *g, int *waited);
/*
* y = w x for a product group, the rows of each task split between the GPU
* and the pool: the GPU takes the first rows, a share that adapts per
* product shape so that both sides finish together. With g NULL, blocks of
* fewer vectors than JANAS_GPU_MIN_N (default 2), or after a GPU failure,
* the pool computes everything. The results are the same in every case.
*/
void janas_gpu_matvec_group(struct janas_gpu *g, struct janas_pool *pool,
const struct janas_matvec_task *tasks, size_t n);
/*
* The fast prompt's product of weights left as they are, f32 (bf16 0) or
* bf16 (1), rows x cols, with n vectors of cols floats in x: y[v * rows + r]
* = scale x row r . vector v, on the GPU alone, waited for. Only for n of
* JANAS_GPU_FAST_N and more, cols a multiple of 16, and w, x and y in memory
* the GPU holds; 0, or -1 with nothing written (the caller computes it).
*/
int janas_gpu_dense_mm(struct janas_gpu *g, int bf16, const void *w,
const float *x, float *y, size_t rows, size_t cols,
size_t n, float scale);
/*
* y = scale w x for bf16 weights (rows x cols, in memory the GPU holds),
* with n vectors of cols floats in x and y[v * rows + r], computed as the
* CPU computes them with AVX2 and FMA (bf16.h): the same bits, so for any
* n the GPU takes (janas_gpu_min_n); x and y in place where the GPU shares
* them with the host, else through its stages. Waited for. Not in a pass
* the tuner gave the CPU (janas_gpu_set_only_fast). 0, or -1 with nothing
* written (the caller computes it). Qwen3.8-Flash-Next's mixes of the
* streams, 1.25 GB of weights a token: a reply's tokens on a card with
* memory of its own.
*/
int janas_gpu_bf16_mv(struct janas_gpu *g, const uint16_t *w, const float *x,
float *y, size_t rows, size_t cols, size_t n,
float scale);
/*
* Whether the fast prompt is on (unless JANAS_GPU_FAST_PROMPT=0): blocks of
* JANAS_GPU_FAST_N vectors and more of the types it has shaders for go to
* the GPU whole, rounded group by group - close to the CPU's results, not
* bit for bit - and, so that a prompt always gives the same results, every
* such block goes to the GPU, whatever the tuner would choose.
*/
/* 64: a mixture's experts get some 16 tokens each of a block of 256, too
few to fill a tile of 64 - forced whole onto the GPU, Qwen3-30B-A3B read
a prompt at 84 tokens/s instead of 97 (9 Oct 2026) */
#define JANAS_GPU_FAST_N 64
int janas_gpu_fast_prompt(void);
/* For the passes that follow: only the fast prompt's products on the GPU,
the rest on the CPU (a pass the tuner gave the CPU alone). */
void janas_gpu_set_only_fast(struct janas_gpu *g, int on);
/* The tokens of the pass that follows. A mixture's experts go to the fast
prompt's shaders (experts_id) only in a prompt's blocks, of
JANAS_GPU_FAST_N tokens and more: a chain of drafts to verify stays bit
for bit with the CPU, so that drafts never change the text. Not said
(a probe calling the products itself): any pass. */
void janas_gpu_set_pass(struct janas_gpu *g, uint32_t n);
/* JANAS_GPU_REPEATABLE=1: a prompt gives the same results every time - a
mixture's experts never on the GPU (janas_m_gpu: an expert's product went
to the fast prompt's tiles where the GPU could read its slot, to the CPU
where it could not, as the cache filled), and in a prompt's blocks the
GPU only the fast prompt's products (janas_m_apply_candidate):
products shared with the CPU, begun beside them, made the GPU busy, and
the fast ones asked meanwhile went to the CPU, as the tuner tried one
configuration or another. */
int janas_gpu_repeatable(void);
/* Keeps the GPU out of its sleep states (an empty dispatch every 0.5 ms,
JANAS_GPU_KEEPALIVE_US) while on: for the passes that use it. */
void janas_gpu_keep_awake(struct janas_gpu *g, int on);
/* The device's name, and the subgroup size the shaders run with. */
const char *janas_gpu_name(const struct janas_gpu *g);
unsigned janas_gpu_subgroup(const struct janas_gpu *g);
/* The subgroup sizes the device lets the shaders be given (the running one
twice where it cannot be chosen): JANAS_GPU_SG picks one at creation. */
void janas_gpu_subgroup_range(const struct janas_gpu *g, unsigned *min,
unsigned *max);
/* One line on the device and its driver for a report: name, kind, driver
and its version, Vulkan, subgroups, the float controls that keep its
results the CPU's. */
const char *janas_gpu_about(const struct janas_gpu *g);
/* What the check of the GPU's products against the CPU's on opening left
out on this driver ("" if nothing; gpu_selfcheck.c). */
const char *janas_gpu_check_note(const struct janas_gpu *g);
/* For diagnosis (gpu_check): the shader that recorded the last product,
as "q6k_row NV 1 SG 32" (its name, vectors a workgroup, subgroup size,
kind where it serves several types), "" if none; and the products'
paths switched on or off, the value before returned. */
size_t janas_gpu_last_shader(const struct janas_gpu *g, char *buf, size_t len);
enum janas_gpu_path {
JANAS_GPU_PATH_ROW, /* the subgroup-per-row shaders (*_row.comp) */
JANAS_GPU_PATH_ROW_NV, /* ... on up to this many vectors (4) */
JANAS_GPU_PATH_MVD, /* the lanes on the vectors (*_mvd.comp) */
JANAS_GPU_PATH_MV /* q4k_mv.comp (off) */
};
int janas_gpu_set_path(struct janas_gpu *g, enum janas_gpu_path path,
int value);
/* The fewest vectors a product needs to be given to the GPU: 1 on a
discrete GPU (single tokens too), 2 on an integrated one;
JANAS_GPU_MIN_N sets it. */
size_t janas_gpu_min_n(const struct janas_gpu *g);
/* ... set by the engine: 1 where the user asked for replies on the GPU
(--reply-on gpu). */
void janas_gpu_set_min_n(struct janas_gpu *g, size_t n);
/* The most vectors the shaders with the lanes on the vectors take
(*_mvd.comp: subgroups of 8 lanes, JANAS_GPU_MVD=0 leaves them out),
beyond which a product goes to the slower shaders of many vectors: 8,
or 0 without them. */
size_t janas_gpu_few_max(const struct janas_gpu *g);
/* Whether the GPU reads and writes the host's memory in place (an
integrated GPU with the host-memory import, not JANAS_GPU_COPY): what the
whole token needs, and what makes the engine's buffers worth importing.
On a card with memory of its own an import is a copy. */
int janas_gpu_shares_host(const struct janas_gpu *g);
/* Bytes of a card's own memory still free for Janas (85% of it, less what
it took); 0 on a GPU that shares the host's memory. */
size_t janas_gpu_local_room(const struct janas_gpu *g);
/* Bytes of host memory and the GPU's copy of them made the same: to the
device (to_device 1) or back to the host (0), for buffers both write on
a card with memory of its own; nothing where the GPU shares the host's
memory. 0, or -1 (not imported, or the GPU busy or broken). */
int janas_gpu_sync(struct janas_gpu *g, void *p, size_t bytes, int to_device);
#endif