Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

10.9 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * gpu.h - an optional integrated GPU for quantized products (Vulkan).
 *
 * Vulkan is loaded at run time: without a driver or a suitable GPU,
 * janas_gpu_create returns NULL and the engine runs on the CPU alone. The
 * GPU reads the model's weights where they are (host memory imported, no
 * copy) and computes with the same arithmetic as the CPU kernels, so a
 * product split between the two gives the same bits as either alone.
 *
 * A product group is split by rows: janas_gpu_begin submits the first rows
 * of each task to the GPU (one submission for the group), the caller
 * computes the other rows on the CPU meanwhile, janas_gpu_finish waits and
 * writes the GPU's rows into the tasks' outputs. One group at a time.
 */
#ifndef JANAS_LLM_GPU_H
#define JANAS_LLM_GPU_H

#include <stddef.h>
#include <stdint.h>

#include "matvec.h"

struct janas_gpu;

/*
 * Q6_K in the GPU's aligned copy (JANAS_GPU_REPACK=1 with the whole token,
 * model_gpu.c): each 210-byte block padded to 224 - ql 128, qh 64, scales
 * 16, d at byte 208 - so that every part starts on 16 bytes. Only the GPU
 * reads it, only for single vectors; a Q6_K_P block with its three planes
 * becomes the Q6_K block it came from.
 */
#define JANAS_Q6_K_A 216
#define JANAS_Q6KA_BLOCK 224

/*
 * The single token's copies in tiles of JANAS_TILE rows (JANAS_GPU_REPACK=2:
 * the Q6_K and the Q4_K matrices with rows of 32 blocks and more; 3: every
 * Q4_K), block b of a tile's rows side by side, so that the subgroups (a
 * row each) read neighbouring memory at the same time: faster for long
 * rows (Qwen3-4B's down matrices), slower for short ones on the
 * development laptop's integrated GPU. Block b of row r is block ((r / T)
 * nb + b) T + r % T; rows padded to a whole tile.
 */
#define JANAS_TILE 64
#define JANAS_Q4_K_T 217
#define JANAS_Q6_K_AT 218

struct janas_gpu *janas_gpu_create(char *err, size_t err_len);
void janas_gpu_destroy(struct janas_gpu *g);

/* Makes host memory readable by the GPU: p and bytes multiples of 4096.
   Returns 0 or -1. */
int janas_gpu_import(struct janas_gpu *g, const void *p, size_t bytes);

/*
 * Memory allocated by the GPU driver, mapped and cached for the host, that
 * the GPU reads with no import: bytes rounded up to 4096, NULL on failure,
 * released by janas_gpu_destroy. After writing it on the CPU, call
 * janas_gpu_flush before the GPU reads it.
 */
void *janas_gpu_alloc(struct janas_gpu *g, size_t bytes);
void janas_gpu_flush(struct janas_gpu *g, const void *p, size_t bytes);

/* Whether the GPU can compute this task (type supported, weights imported,
   sizes within its buffers). */
int janas_gpu_can(const struct janas_gpu *g, const struct janas_matvec_task *t);

/*
 * Submits rows [0, gpu_rows[i]) of each task (tasks the GPU cannot compute
 * must have gpu_rows[i] = 0). Returns 0, or -1 with nothing submitted.
 */
int janas_gpu_begin(struct janas_gpu *g, const struct janas_matvec_task *tasks,
                    const size_t *gpu_rows, size_t n);

/*
 * Experts on the GPU, bit for bit as the CPU computes them: for each expert
 * i, gate_i x and up_i x (ff rows), h = SiLU(gate) * up quantized to Q8_K
 * (the deterministic functions of vmath.h and quant.c), then down_i h (d
 * rows), stored at out + i * d by janas_gpu_finish. x is one Q8_K vector of
 * d values. One submission; the weights must be in shared or copied
 * regions. Returns 0, or -1 with nothing submitted.
 */
struct janas_gpu_expert {
    const void *gate, *up, *down;
    int gate_type, up_type, down_type; /* JANAS_Q4_K or JANAS_Q6_K */
};
int janas_gpu_ffn_begin(struct janas_gpu *g, const struct janas_gpu_expert *e,
                        size_t n, const struct janas_block_q8k *x, size_t d,
                        size_t ff, float *out);

/* Waits for the submitted rows and stores them. Returns 0 or -1. Sets
 *waited to whether the GPU was still running when called. */
int janas_gpu_finish(struct janas_gpu *g, int *waited);

/*
 * y = w x for a product group, the rows of each task split between the GPU
 * and the pool: the GPU takes the first rows, a share that adapts per
 * product shape so that both sides finish together. With g NULL, blocks of
 * fewer vectors than JANAS_GPU_MIN_N (default 2), or after a GPU failure,
 * the pool computes everything. The results are the same in every case.
 */
void janas_gpu_matvec_group(struct janas_gpu *g, struct janas_pool *pool,
                            const struct janas_matvec_task *tasks, size_t n);

/*
 * The fast prompt's product of weights left as they are, f32 (bf16 0) or
 * bf16 (1), rows x cols, with n vectors of cols floats in x: y[v * rows + r]
 * = scale x row r . vector v, on the GPU alone, waited for. Only for n of
 * JANAS_GPU_FAST_N and more, cols a multiple of 16, and w, x and y in memory
 * the GPU holds; 0, or -1 with nothing written (the caller computes it).
 */
int janas_gpu_dense_mm(struct janas_gpu *g, int bf16, const void *w,
                       const float *x, float *y, size_t rows, size_t cols,
                       size_t n, float scale);

/*
 * y = scale w x for bf16 weights (rows x cols, in memory the GPU holds),
 * with n vectors of cols floats in x and y[v * rows + r], computed as the
 * CPU computes them with AVX2 and FMA (bf16.h): the same bits, so for any
 * n the GPU takes (janas_gpu_min_n); x and y in place where the GPU shares
 * them with the host, else through its stages. Waited for. Not in a pass
 * the tuner gave the CPU (janas_gpu_set_only_fast). 0, or -1 with nothing
 * written (the caller computes it). Qwen3.8-Flash-Next's mixes of the
 * streams, 1.25 GB of weights a token: a reply's tokens on a card with
 * memory of its own.
 */
int janas_gpu_bf16_mv(struct janas_gpu *g, const uint16_t *w, const float *x,
                      float *y, size_t rows, size_t cols, size_t n,
                      float scale);

/*
 * Whether the fast prompt is on (unless JANAS_GPU_FAST_PROMPT=0): blocks of
 * JANAS_GPU_FAST_N vectors and more of the types it has shaders for go to
 * the GPU whole, rounded group by group - close to the CPU's results, not
 * bit for bit - and, so that a prompt always gives the same results, every
 * such block goes to the GPU, whatever the tuner would choose.
 */
/* 64: a mixture's experts get some 16 tokens each of a block of 256, too
   few to fill a tile of 64 - forced whole onto the GPU, Qwen3-30B-A3B read
   a prompt at 84 tokens/s instead of 97 (9 Oct 2026) */
#define JANAS_GPU_FAST_N 64
int janas_gpu_fast_prompt(void);
/* For the passes that follow: only the fast prompt's products on the GPU,
   the rest on the CPU (a pass the tuner gave the CPU alone). */
void janas_gpu_set_only_fast(struct janas_gpu *g, int on);
/* The tokens of the pass that follows. A mixture's experts go to the fast
   prompt's shaders (experts_id) only in a prompt's blocks, of
   JANAS_GPU_FAST_N tokens and more: a chain of drafts to verify stays bit
   for bit with the CPU, so that drafts never change the text. Not said
   (a probe calling the products itself): any pass. */
void janas_gpu_set_pass(struct janas_gpu *g, uint32_t n);
/* JANAS_GPU_REPEATABLE=1: a prompt gives the same results every time - a
   mixture's experts never on the GPU (janas_m_gpu: an expert's product went
   to the fast prompt's tiles where the GPU could read its slot, to the CPU
   where it could not, as the cache filled), and in a prompt's blocks the
   GPU only the fast prompt's products (janas_m_apply_candidate):
   products shared with the CPU, begun beside them, made the GPU busy, and
   the fast ones asked meanwhile went to the CPU, as the tuner tried one
   configuration or another. */
int janas_gpu_repeatable(void);

/* Keeps the GPU out of its sleep states (an empty dispatch every 0.5 ms,
   JANAS_GPU_KEEPALIVE_US) while on: for the passes that use it. */
void janas_gpu_keep_awake(struct janas_gpu *g, int on);

/* The device's name, and the subgroup size the shaders run with. */
const char *janas_gpu_name(const struct janas_gpu *g);
unsigned janas_gpu_subgroup(const struct janas_gpu *g);
/* The subgroup sizes the device lets the shaders be given (the running one
   twice where it cannot be chosen): JANAS_GPU_SG picks one at creation. */
void janas_gpu_subgroup_range(const struct janas_gpu *g, unsigned *min,
                              unsigned *max);
/* One line on the device and its driver for a report: name, kind, driver
   and its version, Vulkan, subgroups, the float controls that keep its
   results the CPU's. */
const char *janas_gpu_about(const struct janas_gpu *g);
/* What the check of the GPU's products against the CPU's on opening left
   out on this driver ("" if nothing; gpu_selfcheck.c). */
const char *janas_gpu_check_note(const struct janas_gpu *g);

/* For diagnosis (gpu_check): the shader that recorded the last product,
   as "q6k_row NV 1 SG 32" (its name, vectors a workgroup, subgroup size,
   kind where it serves several types), "" if none; and the products'
   paths switched on or off, the value before returned. */
size_t janas_gpu_last_shader(const struct janas_gpu *g, char *buf, size_t len);
enum janas_gpu_path {
    JANAS_GPU_PATH_ROW,    /* the subgroup-per-row shaders (*_row.comp) */
    JANAS_GPU_PATH_ROW_NV, /* ... on up to this many vectors (4) */
    JANAS_GPU_PATH_MVD,    /* the lanes on the vectors (*_mvd.comp) */
    JANAS_GPU_PATH_MV      /* q4k_mv.comp (off) */
};
int janas_gpu_set_path(struct janas_gpu *g, enum janas_gpu_path path,
                       int value);
/* The fewest vectors a product needs to be given to the GPU: 1 on a
   discrete GPU (single tokens too), 2 on an integrated one;
   JANAS_GPU_MIN_N sets it. */
size_t janas_gpu_min_n(const struct janas_gpu *g);
/* ... set by the engine: 1 where the user asked for replies on the GPU
   (--reply-on gpu). */
void janas_gpu_set_min_n(struct janas_gpu *g, size_t n);
/* The most vectors the shaders with the lanes on the vectors take
   (*_mvd.comp: subgroups of 8 lanes, JANAS_GPU_MVD=0 leaves them out),
   beyond which a product goes to the slower shaders of many vectors: 8,
   or 0 without them. */
size_t janas_gpu_few_max(const struct janas_gpu *g);
/* Whether the GPU reads and writes the host's memory in place (an
   integrated GPU with the host-memory import, not JANAS_GPU_COPY): what the
   whole token needs, and what makes the engine's buffers worth importing.
   On a card with memory of its own an import is a copy. */
int janas_gpu_shares_host(const struct janas_gpu *g);

/* Bytes of a card's own memory still free for Janas (85% of it, less what
   it took); 0 on a GPU that shares the host's memory. */
size_t janas_gpu_local_room(const struct janas_gpu *g);
/* Bytes of host memory and the GPU's copy of them made the same: to the
   device (to_device 1) or back to the host (0), for buffers both write on
   a card with memory of its own; nothing where the GPU shares the host's
   memory. 0, or -1 (not imported, or the GPU busy or broken). */
int janas_gpu_sync(struct janas_gpu *g, void *p, size_t bytes, int to_device);

#endif