37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
Release 2026-09-28: more of the work on the GPU
camauri97149f48ba7d
diff --git a/ChangeLog.md b/ChangeLog.mdindex e45d916..b4052c9 100644--- a/ChangeLog.md+++ b/ChangeLog.md@@ -2,6 +2,29 @@> Curated, user-facing summary of completed work, newest first.+## [2026-09-28] - More of the work on the GPU++- **The experts' weights go to the GPU too** once every expert of every+ layer is in the cache: the feed-forward weights of a dense model, and a+ mixture small enough to fit. They were always left to the CPU. On the+ development laptop's integrated GPU, Qwen3-4B reads a prompt 29% faster+ (83 to 107 tokens/s) and the chip spends 36% less energy per token+ (0.49 to 0.31 J, RAPL package counter); replies are unchanged.+ `JANAS_GPU_EXPERTS=0` keeps them on the CPU. Regions over 4 GB no longer+ turn the GPU off: they are imported in pieces.+- **Single tokens on a discrete GPU**, where the tuner finds it faster. The+ tuner never gave a reply's single token to the GPU; on an RTX 5080 (issue+ #15) doing so made replies 11-16% faster, even with only attention and+ output in the card's memory. A discrete GPU now takes them, and the tuner+ tries it and keeps it only if it measures faster; integrated GPUs are+ left as they were.+- **Two more weight types on the GPU**: Q6_K in bit planes (the down+ matrices of every Qwen mixture and of some layers of the dense models)+ and Q5_K (Qwen3.5 2B and 9B). Like the others they give the CPU's+ results bit for bit, so a product still splits between the two.+- `JANAS_GPU_VERBOSE=2` lists each product shape once with its weight+ types and whether the GPU takes them.+## [2026-09-27] - No Python left- The README names its author, Maurizio "camauri" Cammalleri, under thediff --git a/README.md b/README.mdindex 1de1b5c..40bf98f 100644--- a/README.md+++ b/README.md@@ -1079,10 +1079,14 @@ and for [a model that will not convert](https://github.com/prabanta-dev/janas/is1. **CPUs without AVX-VNNI**, and **AMD** — the arithmetic has a path for themand it is the least exercised of all.-2. **Discrete GPUs** (NVIDIA, AMD, Intel Arc). The GPU support is written and- correct, bit for bit, but on an integrated GPU it gains nothing: whether it- pays on a real card is unknown. **Careful:** a GPU driver reset takes the- desktop with it. Start with `janas-bench`, not with a long chat.+2. **Discrete GPUs** (NVIDIA, AMD, Intel Arc). The GPU support is correct,+ bit for bit, and it pays where it was measured: on the development+ laptop's integrated GPU a dense model whose weights all fit in memory+ reads a prompt about 29% faster (Qwen3-4B, 83 to 107 tokens/s), and on an+ RTX 5080 (issue #15) Qwen3-4B read 13% faster and, with the GPU on single+ tokens too, replied 11-16% faster. Every other card is unknown.+ **Careful:** a GPU driver reset takes the desktop with it. Start with+ `janas-bench`, not with a long chat.3. **Machines with 16 GB or less.** The engine is supposed to trade quality formemory on its own — fewer bits per weight, fewer experts per token — and thethresholds for that were measured on 32 GB.diff --git a/src/llm/expert_cache.c b/src/llm/expert_cache.cindex acb1a41..6577cb0 100644--- a/src/llm/expert_cache.c+++ b/src/llm/expert_cache.c@@ -74,8 +74,9 @@ struct janas_expert_cache {/* one request or one batch of the background filling at a time */pthread_mutex_t api_lock;- size_t filled; /* slots that hold an expert */- double last_request; /* when the caller last asked for experts */+ size_t filled; /* slots that hold an expert */+ _Atomic uint64_t loads; /* experts ever put in a slot */+ double last_request; /* when the caller last asked for experts *//* the background filling of the cache, if it was started */pthread_t warmer;int has_warmer, warm_stop;@@ -383,6 +384,7 @@ static int32_t load(struct janas_expert_cache *c, uint32_t layer, uint32_t e)} else {c->filled++;}+ atomic_fetch_add(&c->loads, 1);int64_t k = (int64_t)layer * c->j->h.n_expert + e;c->key[s] = k;c->where[k] = s;@@ -662,6 +664,34 @@ int janas_expert_cache_warm_background(struct janas_expert_cache *c,return 0;}+const uint8_t *janas_expert_cache_resident(struct janas_expert_cache *c,+ uint64_t *bytes, uint64_t *loads)+{+ size_t total = (size_t)c->j->h.n_layer * c->j->h.n_expert;+ /* between begin and finish the caller holds api_lock: asked there, the+ answer is no rather than waiting on itself (outstanding is the+ caller's own) */+ if (c->n_slots < total || c->outstanding)+ return NULL;+ pthread_mutex_lock(&c->api_lock);+ int busy = c->has_warmer && !atomic_load(&c->warm_over);+ pthread_mutex_lock(&c->lock);+ busy |= c->pending;+ pthread_mutex_unlock(&c->lock);+ int full = c->filled >= total;+ *loads = atomic_load(&c->loads);+ pthread_mutex_unlock(&c->api_lock);+ if (busy || !full)+ return NULL;+ *bytes = (uint64_t)c->n_slots * c->slot_bytes;+ return c->arena;+}++uint64_t janas_expert_cache_loads(const struct janas_expert_cache *c)+{+ return atomic_load(&((struct janas_expert_cache *)c)->loads);+}+int janas_expert_cache_warming(const struct janas_expert_cache *c){if (!c->has_warmer)diff --git a/src/llm/expert_cache.h b/src/llm/expert_cache.hindex a1cd022..d7592ee 100644--- a/src/llm/expert_cache.h+++ b/src/llm/expert_cache.h@@ -53,6 +53,17 @@ void janas_expert_cache_destroy(struct janas_expert_cache *c);size_t janas_expert_cache_slots(const struct janas_expert_cache *c);+/*+ * The arena, when it holds every expert of every layer and nothing is+ * being read into it: from then on no slot changes (there is nothing left+ * to load), so a copy of it stays true - for a GPU. NULL otherwise, and+ * between begin and finish. loads: the experts ever put in a slot, to tell+ * later whether one changed.+ */+const uint8_t *janas_expert_cache_resident(struct janas_expert_cache *c,+ uint64_t *bytes, uint64_t *loads);+uint64_t janas_expert_cache_loads(const struct janas_expert_cache *c);+/** Loads up to the cache capacity the experts with the highest counts* (counts has n_layer * n_expert entries, layer-major; zero means never).diff --git a/src/llm/gpu.c b/src/llm/gpu.cindex 5313038..524f839 100644--- a/src/llm/gpu.c+++ b/src/llm/gpu.c@@ -24,11 +24,13 @@#include <unistd.h>#include <vulkan/vulkan.h>-#include "q4k_matvec.h" /* janas_spv_q4k_matvec, generated by build.sh */-#include "q6k_matvec.h" /* janas_spv_q6k_matvec */-#include "expert_act.h" /* janas_spv_expert_act */+#include "q4k_matvec.h" /* janas_spv_q4k_matvec, generated by build.sh */+#include "q6k_matvec.h" /* janas_spv_q6k_matvec */+#include "q6kp_matvec.h" /* janas_spv_q6kp_matvec */+#include "q5k_matvec.h" /* janas_spv_q5k_matvec */+#include "expert_act.h" /* janas_spv_expert_act */-#define MAX_REGIONS 8+#define MAX_REGIONS 32#define MAX_PENDING 64#define MAX_XCACHE 8#define X_STAGE_BYTES ((size_t)8 << 20)@@ -114,10 +116,12 @@ struct pending {size_t off; /* in the Y stage */};-/* The push constants of the shader. */+/* The push constants of the shaders (level and the bare planes: Q6_K_P+ only). */struct push {uint32_t w[2], x[2], y[2];- uint32_t rows, nb, nv, xs, ys;+ uint32_t rows, nb, nv, xs, ys, level;+ uint32_t p1[2], p0[2];};struct janas_gpu {@@ -131,9 +135,11 @@ struct janas_gpu {VkPhysicalDevice pd;VkDevice dev;VkQueue queue;- VkShaderModule sm, sm6, sma;+ VkShaderModule sm, sm6, sm6p, sm5, sma;VkPipelineLayout pl;- VkPipeline pipe, pipe6, pipea; /* Q4_K, Q6_K, expert activation */+ VkPipeline pipe, pipe6, pipe6p, pipe5, pipea; /* Q4_K, Q6_K, Q6_K_P,+ Q5_K, experts'+ activation */VkCommandPool cp;VkCommandBuffer cb;VkFence fence;@@ -156,7 +162,8 @@ struct janas_gpu {uint32_t calls, waits;double t_begin, t_cpu, t_wait, t_copy; /* seconds, verbose only */uint64_t rows_gpu, rows_all;- uint32_t off; /* calls left on the CPU alone */+ uint32_t off; /* calls left on the CPU alone */+ uint32_t seen; /* its tasks printed (JANAS_GPU_VERBOSE=2) */} share[SHARE_SLOTS];size_t min_n;size_t full_n; /* from this many vectors the GPU takes every row */@@ -582,6 +589,12 @@ struct janas_gpu *janas_gpu_create(char *err, size_t err_len)(r = make_pipeline(g, janas_spv_q6k_matvec,sizeof(janas_spv_q6k_matvec), &g->sm6, &g->pipe6)) !=VK_SUCCESS ||+ (r = make_pipeline(g, janas_spv_q6kp_matvec,+ sizeof(janas_spv_q6kp_matvec), &g->sm6p,+ &g->pipe6p)) != VK_SUCCESS ||+ (r = make_pipeline(g, janas_spv_q5k_matvec,+ sizeof(janas_spv_q5k_matvec), &g->sm5, &g->pipe5)) !=+ VK_SUCCESS ||(r = make_pipeline(g, janas_spv_expert_act,sizeof(janas_spv_expert_act), &g->sma, &g->pipea)) !=VK_SUCCESS) {@@ -613,12 +626,16 @@ struct janas_gpu *janas_gpu_create(char *err, size_t err_len)if (g->keep_us > 0 && record_keep_alive(g) == 0 &&pthread_create(&g->ka, NULL, keep_alive, g) == 0)g->ka_running = 1;+ /* single tokens too on a card with memory of its own: there the GPU+ adds its bandwidth to the RAM's (measured on an RTX 5080, issue #15:+ replies 11-16% faster with only attention and output in its memory);+ an integrated GPU shares the RAM's, and lost 6% */const char *e = getenv("JANAS_GPU_MIN_N");- g->min_n = e && atoi(e) > 0 ? (size_t)atoi(e) : 2;+ g->min_n = e && atoi(e) > 0 ? (size_t)atoi(e) : g->discrete ? 1 : 2;e = getenv("JANAS_GPU_FULL_N");g->full_n = e && atoi(e) > 0 ? (size_t)atoi(e) : (size_t)-1;e = getenv("JANAS_GPU_VERBOSE");- g->verbose = e && atoi(e) > 0;+ g->verbose = e && atoi(e) > 0 ? atoi(e) : 0;g->nospin = getenv("JANAS_GPU_NOSPIN") != NULL;g->block_wait = getenv("JANAS_GPU_BLOCKWAIT") != NULL;return g;@@ -682,12 +699,20 @@ void janas_gpu_destroy(struct janas_gpu *g)g->vkDestroyPipeline(g->dev, g->pipe, NULL);if (g->pipe6)g->vkDestroyPipeline(g->dev, g->pipe6, NULL);+ if (g->pipe6p)+ g->vkDestroyPipeline(g->dev, g->pipe6p, NULL);+ if (g->pipe5)+ g->vkDestroyPipeline(g->dev, g->pipe5, NULL);if (g->pipea)g->vkDestroyPipeline(g->dev, g->pipea, NULL);if (g->sma)g->vkDestroyShaderModule(g->dev, g->sma, NULL);if (g->sm6)g->vkDestroyShaderModule(g->dev, g->sm6, NULL);+ if (g->sm6p)+ g->vkDestroyShaderModule(g->dev, g->sm6p, NULL);+ if (g->sm5)+ g->vkDestroyShaderModule(g->dev, g->sm5, NULL);if (g->pl)g->vkDestroyPipelineLayout(g->dev, g->pl, NULL);if (g->sm)@@ -711,6 +736,11 @@ unsigned janas_gpu_subgroup(const struct janas_gpu *g)return g->sg;}+size_t janas_gpu_min_n(const struct janas_gpu *g)+{+ return g->min_n;+}+void janas_gpu_keep_awake(struct janas_gpu *g, int on){if (g)@@ -971,20 +1001,40 @@ static const struct region *find_region(const struct janas_gpu *g,return NULL;}-/* Bytes the shader may read for rows of a task: Q6_K reads up to 2 bytes- past the last block (whole words). */+/* Bytes the shader may read for rows of a task: Q6_K and the high plane of+ Q6_K_P read up to 2 bytes past the last block (whole words). */static size_t read_bytes(const struct janas_matvec_task *t, size_t rows){size_t nb = t->cols / JANAS_QK;return rows * nb * janas_qtype_block_size(t->type) +- (t->type == JANAS_Q6_K ? 4 : 0);+ (t->type == JANAS_Q6_K || t->type == JANAS_Q6_K_P ? 4 : 0);+}++/* The bare planes of a Q6_K_P task: level 3 (both), 2 (plane[0]) or 1. */+static int plane_level(const struct janas_matvec_task *t)+{+ return t->plane[0] && t->plane[1] ? 3 : t->plane[0] ? 2 : 1;+}++/* The region holding rows of a bare plane (4-byte aligned), or NULL. */+static const struct region *plane_region(const struct janas_gpu *g,+ const void *p, size_t rows, size_t nb)+{+ return (uintptr_t)p % 4 == 0+ ? find_region(g, p, rows * nb * JANAS_Q6KP_PLANE)+ : NULL;}int janas_gpu_can(const struct janas_gpu *g, const struct janas_matvec_task *t){size_t nb = t->cols / JANAS_QK, nv = t->n_vec ? t->n_vec : 1;- int typed = (t->type == JANAS_Q4_K && (uintptr_t)t->w % 16 == 0) ||- (t->type == JANAS_Q6_K && (uintptr_t)t->w % 4 == 0);+ int typed =+ (t->type == JANAS_Q4_K && (uintptr_t)t->w % 16 == 0) ||+ (t->type == JANAS_Q5_K && (uintptr_t)t->w % 16 == 0) ||+ (t->type == JANAS_Q6_K && (uintptr_t)t->w % 4 == 0) ||+ (t->type == JANAS_Q6_K_P && (uintptr_t)t->w % 4 == 0 &&+ (plane_level(t) < 2 || plane_region(g, t->plane[0], t->rows, nb)) &&+ (plane_level(t) < 3 || plane_region(g, t->plane[1], t->rows, nb)));return !g->broken && typed && t->cols % JANAS_QK == 0 &&nv * nb * GPU_BLOCK <= X_STAGE_BYTES &&t->rows * nv * sizeof(float) <= Y_STAGE_BYTES &&@@ -1006,6 +1056,8 @@ static void convert_x(uint8_t *dst, const struct janas_block_q8k *x, size_t xs,}}+static void push_addr(uint32_t out[2], VkDeviceAddress a);+int janas_gpu_begin(struct janas_gpu *g, const struct janas_matvec_task *tasks,const size_t *gpu_rows, size_t n){@@ -1030,7 +1082,10 @@ int janas_gpu_begin(struct janas_gpu *g, const struct janas_matvec_task *tasks,if (gpu_rows[i] == 0)continue;const struct janas_matvec_task *t = &tasks[i];- VkPipeline want = t->type == JANAS_Q6_K ? g->pipe6 : g->pipe;+ VkPipeline want = t->type == JANAS_Q6_K ? g->pipe6+ : t->type == JANAS_Q6_K_P ? g->pipe6p+ : t->type == JANAS_Q5_K ? g->pipe5+ : g->pipe;if (want != bound) {g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE, want);bound = want;@@ -1073,6 +1128,19 @@ int janas_gpu_begin(struct janas_gpu *g, const struct janas_matvec_task *tasks,.nv = (uint32_t)nv,.xs = (uint32_t)nb,.ys = (uint32_t)gpu_rows[i]};+ if (t->type == JANAS_Q6_K_P) {+ pc.level = (uint32_t)plane_level(t);+ for (int q = 0; q < (int)pc.level - 1; q++) {+ const struct region *pr =+ plane_region(g, t->plane[q], gpu_rows[i], nb);+ if (!pr)+ return -1;+ push_addr(q == 0 ? pc.p1 : pc.p0,+ pr->addr ++ (VkDeviceAddress)((const uint8_t *)t->plane[q] -+ pr->host));+ }+ }g->vkCmdPushConstants(g->cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,sizeof(pc), &pc);g->vkCmdDispatch(@@ -1331,6 +1399,16 @@ void janas_gpu_matvec_group(struct janas_gpu *g, struct janas_pool *pool,uint64_t key = (uint64_t)n << 48 ^ (uint64_t)tasks[0].rows << 24 ^(uint64_t)tasks[0].cols << 8 ^ nv;struct share *sh = find_share(g, key);+ /* JANAS_GPU_VERBOSE=2: each shape once, with its tasks' types and+ whether the GPU can take them */+ if (g->verbose > 1 && sh && sh->calls == 0 && !sh->seen) {+ sh->seen = 1;+ fprintf(stderr, "gpu shape %016llx:", (unsigned long long)key);+ for (size_t i = 0; i < n; i++)+ fprintf(stderr, " type %d %zux%zu%s", tasks[i].type, tasks[i].rows,+ tasks[i].cols, janas_gpu_can(g, &tasks[i]) ? "" : " (CPU)");+ fprintf(stderr, "\n");+ }if (sh && sh->off > 0) {/* the GPU was late even with the smallest share: CPU for a while */sh->off--;@@ -1356,6 +1434,10 @@ void janas_gpu_matvec_group(struct janas_gpu *g, struct janas_pool *pool,cpu[i] = *t;cpu[i].w =(const uint8_t *)t->w + r * nb * janas_qtype_block_size(t->type);+ for (int q = 0; q < 2; q++) /* Q6_K_P: its bare planes follow */+ if (t->plane[q])+ cpu[i].plane[q] =+ (const uint8_t *)t->plane[q] + r * nb * JANAS_Q6KP_PLANE;cpu[i].y = t->y + r;cpu[i].rows = t->rows - r;cpu[i].y_stride = t->y_stride ? t->y_stride : t->rows;@@ -1514,6 +1596,12 @@ unsigned janas_gpu_subgroup(const struct janas_gpu *g)return 0;}+size_t janas_gpu_min_n(const struct janas_gpu *g)+{+ (void)g;+ return (size_t)-1;+}+void janas_gpu_matvec_group(struct janas_gpu *g, struct janas_pool *pool,const struct janas_matvec_task *tasks, size_t n){diff --git a/src/llm/gpu.h b/src/llm/gpu.hindex d22a150..24cc9a1 100644--- a/src/llm/gpu.h+++ b/src/llm/gpu.h@@ -87,5 +87,9 @@ void janas_gpu_keep_awake(struct janas_gpu *g, int on);/* The device's name, and the subgroup size the shaders run with. */const char *janas_gpu_name(const struct janas_gpu *g);unsigned janas_gpu_subgroup(const struct janas_gpu *g);+/* The fewest vectors a product needs to be given to the GPU: 1 on a+ discrete GPU (single tokens too), 2 on an integrated one;+ JANAS_GPU_MIN_N sets it. */+size_t janas_gpu_min_n(const struct janas_gpu *g);#endifdiff --git a/src/llm/model.c b/src/llm/model.cindex a48f089..fbd4a9f 100644--- a/src/llm/model.c+++ b/src/llm/model.c@@ -353,6 +353,7 @@ int janas_llm_model_forward(struct janas_llm_model *m, const int32_t *tokens,from them, or a slow start decides the whole session */if (r == 0 && dt > 0 && !janas_llm_model_settling(m))janas_tuner_record(&m->tuner, cls, cand, dt / n);+ janas_m_arena_to_gpu(m);return r;}diff --git a/src/llm/model_ffn.c b/src/llm/model_ffn.cindex 9995ef9..664c7cd 100644--- a/src/llm/model_ffn.c+++ b/src/llm/model_ffn.c@@ -253,14 +253,14 @@ static void run_experts(struct janas_llm_model *m,}if (janas_m_exp_tracing())ex_setup += now() - t_enter;- EX_T(ex_t1,- janas_gpu_matvec_group(m->gpu_off || !m->gpu_use ? NULL : m->gpu,- m->compute, t1, 2 * (size_t)n_runs));+ EX_T(ex_t1, janas_gpu_matvec_group(+ m->gpu_off || !m->gpu_use || m->arena_stale ? NULL : m->gpu,+ m->compute, t1, 2 * (size_t)n_runs));struct act_job aj = {m, ng};EX_T(ex_act, janas_pool_run(m->compute, act_worker, &aj));- EX_T(ex_t2,- janas_gpu_matvec_group(m->gpu_off || !m->gpu_use ? NULL : m->gpu,- m->compute, t2, n_runs));+ EX_T(ex_t2, janas_gpu_matvec_group(+ m->gpu_off || !m->gpu_use || m->arena_stale ? NULL : m->gpu,+ m->compute, t2, n_runs));}/* The router's choice for token j: softmax, top-k, normalized weights. */diff --git a/src/llm/model_impl.h b/src/llm/model_impl.hindex 571a745..789e456 100644--- a/src/llm/model_impl.h+++ b/src/llm/model_impl.h@@ -84,7 +84,13 @@ struct janas_llm_model {struct janas_gpu *gpu; /* optional, for products over blocks */int resident_gpu; /* the resident region belongs to the GPU */int gpu_off; /* weights not shared with it: CPU only */- char gpu_why[160]; /* no GPU: why (janas_llm_model_gpu_why) */+ /* the experts' arena on the GPU (dense models, and mixtures whose+ every expert fits the cache): 0 not yet, 1 there, -1 not possible;+ arena_stale when a slot changed since - the GPU's copy no longer+ true, the experts' products stay on the CPU */+ int arena_gpu, arena_stale;+ uint64_t arena_loads;+ char gpu_why[160]; /* no GPU: why (janas_llm_model_gpu_why) */struct janas_expert_cache *cache;const struct janas_arch *a; /* what this architecture has (arch.h) */@@ -324,6 +330,9 @@ struct tok_job {};void janas_m_init_tuner(struct janas_llm_model *m);+int janas_m_gpu_import(struct janas_llm_model *m, const uint8_t *p,+ uint64_t bytes, uint64_t unit);+void janas_m_arena_to_gpu(struct janas_llm_model *m);void janas_m_apply_candidate(struct janas_llm_model *m, int cand);uint32_t *janas_m_profile_read(const struct janas_llm_model *m, size_t *used);void janas_m_profile_write(const struct janas_llm_model *m);diff --git a/src/llm/model_load.c b/src/llm/model_load.cindex 0d093d0..8c16264 100644--- a/src/llm/model_load.c+++ b/src/llm/model_load.c@@ -1039,7 +1039,7 @@ struct janas_llm_model *janas_llm_model_load(const char *path,flushed out of the CPU's caches before the GPU reads it */if (m->resident_gpu)janas_gpu_flush(m->gpu, m->resident, m->j.h.resident_bytes);- else if (janas_gpu_import(m->gpu, m->resident, m->j.h.resident_bytes) !=+ else if (janas_m_gpu_import(m, m->resident, m->j.h.resident_bytes, 0) !=0)m->gpu_off = 1;if (m->mtp_res && janas_gpu_import(m->gpu, m->mtp_res,@@ -1226,6 +1226,78 @@ uint64_t janas_llm_model_warm(const struct janas_llm_model *m, double *seconds)return m->warm_experts;}+/* The GPU region of 4 GB at the most: cut at a multiple of unit (an+ expert slot), or for the resident region (unit 0) where a tensor+ begins, so that no tensor straddles two and each stays in reach */+#define GPU_REGION_MAX ((uint64_t)4 << 30)++static uint64_t resident_cut(const struct janas_llm_model *m, uint64_t from)+{+ uint64_t base = m->j.h.resident_offset, best = from;+ for (uint64_t i = 0; i < m->j.h.n_tensor; i++) {+ uint64_t at = m->j.tensors[i].offset - base;+ if (at > from && at <= from + GPU_REGION_MAX && at > best)+ best = at;+ }+ return best > from ? best : from + GPU_REGION_MAX;+}++int janas_m_gpu_import(struct janas_llm_model *m, const uint8_t *p,+ uint64_t bytes, uint64_t unit)+{+ for (uint64_t at = 0; at < bytes;) {+ uint64_t end = at + GPU_REGION_MAX;+ if (end >= bytes)+ end = bytes;+ else if (unit)+ end = at + GPU_REGION_MAX / unit * unit;+ else+ end = resident_cut(m, at);+ if (end > bytes)+ end = bytes;+ /* the regions are whole pages: the last one's length rounded up+ (the allocations are) */+ uint64_t len = (end - at + 4095) & ~(uint64_t)4095;+ if (janas_gpu_import(m->gpu, p + at, len) != 0)+ return -1;+ at = end;+ }+ return 0;+}++/*+ * The experts on the GPU too: once the cache holds every expert of every+ * layer and nothing is being read (a dense model's feed-forwards, or a+ * mixture that fits), its arena is shared with the GPU or copied to its+ * memory, and the experts' products are split between the two like the+ * others. Checked after each pass until then; one change of a slot later+ * (a new level of bits) and the GPU's copy is no longer used.+ */+void janas_m_arena_to_gpu(struct janas_llm_model *m)+{+ if (!m->gpu || m->gpu_off || !m->cache || m->arena_gpu < 0)+ return;+ if (m->arena_gpu == 1) {+ if (!m->arena_stale &&+ janas_expert_cache_loads(m->cache) != m->arena_loads)+ m->arena_stale = 1;+ return;+ }+ uint64_t bytes, loads;+ const uint8_t *a = janas_expert_cache_resident(m->cache, &bytes, &loads);+ if (!a)+ return;+ uint64_t slot = janas_expert_cache_slot_bytes(m->cache);+ if (getenv("JANAS_GPU_EXPERTS") && atoi(getenv("JANAS_GPU_EXPERTS")) == 0)+ m->arena_gpu = -1;+ else if (janas_m_gpu_import(m, a, bytes, slot) == 0) {+ m->arena_gpu = 1;+ m->arena_loads = loads;+ } else {+ m->arena_gpu = -1; /* no room for it: the CPU goes on alone */+ }+}+int janas_llm_model_settling(const struct janas_llm_model *m){return m->cache && janas_expert_cache_warming(m->cache);diff --git a/src/llm/model_tune.c b/src/llm/model_tune.cindex 4a78445..10a3944 100644--- a/src/llm/model_tune.c+++ b/src/llm/model_tune.c@@ -41,7 +41,9 @@* JANAS_TUNE_THREADS=0 keeps every core without trying the others;* JANAS_TUNE=0 turns tuning off; JANAS_BLOCK_THREADS fixes the threads for* blocks (and turns it off too); JANAS_GPU_ALWAYS=1 makes the GPU variants- * the defaults for several tokens.+ * the defaults for several tokens. The GPU variants are tried for one token+ * too where the GPU takes single tokens (a discrete GPU, or+ * JANAS_GPU_MIN_N=1): the default stays without it.*/void janas_m_init_tuner(struct janas_llm_model *m){@@ -82,14 +84,21 @@ void janas_m_init_tuner(struct janas_llm_model *m)}janas_tuner_init(&m->tuner, c, n, def, (tn && atoi(tn) == 0) || bt);janas_tuner_allow_gpu(&m->tuner, 1);- /* the saved choices of this machine and model */+ /* a GPU that takes single tokens (a discrete one) is tried for replies+ too, and kept only if it measures faster */+ int one = m->gpu && !m->gpu_off && janas_gpu_min_n(m->gpu) <= 1;+ janas_tuner_allow_gpu_one(&m->tuner, one);+ /* the saved choices of this machine and model; "1g" in the key when+ the GPU may take replies, so that choices saved without it do not+ stand in for the tries */char cpu[128], name[128] = "";janas_cpu_name(cpu, sizeof(cpu));janas_jns_meta_str(&m->j, "general.name", name, sizeof(name));- int w = snprintf(m->tune_key, sizeof(m->tune_key), "v2|%s|%d/%d/%d|%s|%s|",- cpu, m->p_cores, m->p_threads, m->all_threads,- m->gpu && !m->gpu_off ? janas_gpu_name(m->gpu) : "no GPU",- name);+ int w =+ snprintf(m->tune_key, sizeof(m->tune_key), "v2|%s|%d/%d/%d|%s%s|%s|",+ cpu, m->p_cores, m->p_threads, m->all_threads,+ m->gpu && !m->gpu_off ? janas_gpu_name(m->gpu) : "no GPU",+ one ? " 1g" : "", name);for (int k = 0; k < n && w > 0 && (size_t)w < sizeof(m->tune_key); k++)w += snprintf(m->tune_key + w, sizeof(m->tune_key) - (size_t)w, "%d%s,",c[k].threads, c[k].gpu ? "g" : "");diff --git a/src/llm/shaders/q5k_matvec.comp b/src/llm/shaders/q5k_matvec.compnew file mode 100644index 0000000..db8536d--- /dev/null+++ b/src/llm/shaders/q5k_matvec.comp@@ -0,0 +1,94 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+#version 450+// Q5_K weights times up to NV Q8_K vectors per workgroup, one row per+// invocation: Q4_K with a fifth bit (qh: bit s of byte l is the fifth bit+// of weight 32 s + l), in the same per-block integer-sum scheme as+// q4k_matvec.comp and janas_q5k_dot_ref, so the results agree with the CPU+// bit for bit. A block is 176 bytes, 11 uvec4: d, dmin and the scales, qh+// in two, the low nibbles in eight. Activations shared across the subgroup+// as in q4k_matvec.comp.+#extension GL_EXT_integer_dot_product : require+#extension GL_KHR_shader_subgroup_basic : require+#extension GL_KHR_shader_subgroup_ballot : require+#extension GL_EXT_buffer_reference : require+#extension GL_EXT_buffer_reference_uvec2 : require++layout(local_size_x = 64) in;+layout(constant_id = 0) const uint NV = 8;+layout(constant_id = 1) const uint SG = 8;+const uint NX = (19u + SG - 1u) / SG;++layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };+layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };++// w: the first row; x: vector v, block b at uvec4 (v * xs + b) * 20;+// y: row r of vector v at y[v * ys + r]+layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys; } p;++int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }+int dp4(uvec4 a, uvec4 b) { return dp(a.x, b.x) + dp(a.y, b.y) + dp(a.z, b.z) + dp(a.w, b.w); }+uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }++void main()+{+ U4 W = U4(p.w), X = U4(p.x);+ F Y = F(p.y);+ uint lane = gl_SubgroupInvocationID;+ uint row = min(gl_GlobalInvocationID.x, p.rows - 1u);+ uint v0 = gl_WorkGroupID.y * NV;+ uint xo[NV];+ for (uint v = 0; v < NV; v++)+ xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;+ float acc[NV], accm[NV];+ for (uint v = 0; v < NV; v++) { acc[v] = 0.0; accm[v] = 0.0; }+ uint wo = row * p.nb * 11u;+ for (uint b = 0; b < p.nb; b++, wo += 11u) {+ uvec4 hd = W.v[wo];+ vec2 dd = unpackHalf2x16(hd.x);+ int sc[8], mn[8];+ for (uint j = 0; j < 4u; j++) {+ sc[j] = int(bitfieldExtract(hd.y, int(8u * j), 6));+ mn[j] = int(bitfieldExtract(hd.z, int(8u * j), 6));+ sc[j + 4u] = int(bitfieldExtract(hd.w, int(8u * j), 4) |+ (bitfieldExtract(hd.y, int(8u * j + 6u), 2) << 4));+ mn[j + 4u] = int(bitfieldExtract(hd.w, int(8u * j + 4u), 4) |+ (bitfieldExtract(hd.z, int(8u * j + 6u), 2) << 4));+ }+ uvec4 qh0 = W.v[wo + 1u], qh1 = W.v[wo + 2u];+ uvec4 q[8];+ for (uint j = 0; j < 8u; j++)+ q[j] = W.v[wo + 3u + j];+ for (uint v = 0; v < NV; v++) {+ uvec4 xr[NX];+ for (uint j = 0; j < NX; j++)+ xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];+ int isum = 0;+ for (uint c = 0; c < 4u; c++) {+ /* sub-blocks 2c (low nibbles) and 2c + 1 (high), each 32+ weights: their fifth bits are bits 2c and 2c + 1 of qh */+ uvec4 q0 = q[2u * c], q1 = q[2u * c + 1u];+ uint s0 = 2u * c, s1 = 2u * c + 1u;+ uvec4 lo0 = (q0 & 0x0f0f0f0fu) | (((qh0 >> s0) & 0x01010101u) << 4);+ uvec4 lo1 = (q1 & 0x0f0f0f0fu) | (((qh1 >> s0) & 0x01010101u) << 4);+ uvec4 hi0 = ((q0 >> 4) & 0x0f0f0f0fu) | (((qh0 >> s1) & 0x01010101u) << 4);+ uvec4 hi1 = ((q1 >> 4) & 0x0f0f0f0fu) | (((qh1 >> s1) & 0x01010101u) << 4);+ isum += (dp4(lo0, bc(xr, 4u * c)) + dp4(lo1, bc(xr, 4u * c + 1u))) * sc[s0] ++ (dp4(hi0, bc(xr, 4u * c + 2u)) + dp4(hi1, bc(xr, 4u * c + 3u))) * sc[s1];+ }+ uvec4 bs0 = bc(xr, 16u), bs1 = bc(xr, 17u);+ float d8 = uintBitsToFloat(bc(xr, 18u).x);+ uint bw[8] = uint[8](bs0.x, bs0.y, bs0.z, bs0.w, bs1.x, bs1.y, bs1.z, bs1.w);+ int imin = 0;+ for (uint s = 0; s < 8u; s++)+ imin += mn[s] * (bitfieldExtract(int(bw[s]), 0, 16) + bitfieldExtract(int(bw[s]), 16, 16));+ acc[v] = fma(dd.x * d8, float(isum), acc[v]);+ accm[v] = fma(dd.y * d8, float(imin), accm[v]);+ xo[v] += 20u;+ }+ }+ if (gl_GlobalInvocationID.x < p.rows)+ for (uint v = 0; v < NV; v++)+ if (v0 + v < p.nv)+ Y.v[(v0 + v) * p.ys + row] = acc[v] - accm[v];+}diff --git a/src/llm/shaders/q6kp_matvec.comp b/src/llm/shaders/q6kp_matvec.compnew file mode 100644index 0000000..dcac1bd--- /dev/null+++ b/src/llm/shaders/q6kp_matvec.comp@@ -0,0 +1,118 @@+/* SPDX-License-Identifier: GPL-3.0-or-later+ Copyright (C) 2026 Maurizio Cammalleri */+#version 450+// Q6_K in bit planes (JANAS_Q6_K_P, quant.h) times up to NV Q8_K vectors,+// one row per invocation, as janas_q6kp_dot_ref computes it: the high plane+// (with the scales and d) in the row's 82-byte blocks, the two bare planes+// in arrays of their own, 64 bytes a block, each packed as Q6_K's qh field.+// At level 3 (all three planes) the weight is the Q6_K one; at 2 and 1 the+// centre of the values the missing planes stand for, folded into the block+// sum as sumi * 2^shk - c * off, the same integers and float operations as+// the CPU: the results agree bit for bit. Activations shared across the+// subgroup as in q4k_matvec.comp. The blocks are 82 bytes, so every other+// one starts 2 bytes off a 4-byte boundary: words are read from the+// boundary below and realigned, as in q6k_matvec.comp.+#extension GL_EXT_integer_dot_product : require+#extension GL_KHR_shader_subgroup_basic : require+#extension GL_KHR_shader_subgroup_ballot : require+#extension GL_EXT_buffer_reference : require+#extension GL_EXT_buffer_reference_uvec2 : require++layout(local_size_x = 64) in;+layout(constant_id = 0) const uint NV = 8;+layout(constant_id = 1) const uint SG = 8;+const uint NX = (19u + SG - 1u) / SG;++layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };+layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };+layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };++// level 3, 2 or 1: the planes present; p1 (level >= 2) and p0 (level 3) the+// bare planes of the first row, nb * 64 bytes a row+layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys, level; uvec2 p1, p0; } p;++int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }+uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }++void main()+{+ U1 W = U1(p.w), P1 = U1(p.p1), P0 = U1(p.p0);+ U4 X = U4(p.x);+ F Y = F(p.y);+ uint lane = gl_SubgroupInvocationID;+ uint row = min(gl_GlobalInvocationID.x, p.rows - 1u);+ uint v0 = gl_WorkGroupID.y * NV;+ uint xo[NV];+ for (uint v = 0; v < NV; v++)+ xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;+ float acc[NV];+ for (uint v = 0; v < NV; v++)+ acc[v] = 0.0;+ uint shk = p.level == 3u ? 1u : p.level == 2u ? 3u : 5u;+ int c = p.level == 3u ? 64 : p.level == 2u ? 61 : 49;+ uint off = row * p.nb * 82u; /* bytes */+ uint pw = row * p.nb * 16u; /* words of the bare planes */+ for (uint b = 0; b < p.nb; b++, off += 82u, pw += 16u) {+ /* the block's 20 words (80 bytes) and d, realigned */+ uint base = off >> 2, sh = (off & 3u) * 8u;+ uint wd[21];+ for (uint k = 0; k < 21u; k++)+ wd[k] = W.v[base + k];+ if (sh != 0u)+ for (uint k = 0; k < 20u; k++)+ wd[k] = (wd[k] >> 16) | (wd[k + 1u] << 16);+ float d = unpackHalf2x16(sh != 0u ? wd[20] >> 16 : wd[20] & 0xffffu).x;+ int sc[16];+ for (uint k = 0; k < 4u; k++)+ for (uint j = 0; j < 4u; j++)+ sc[4u * k + j] = bitfieldExtract(int(wd[16u + k]), int(8u * j), 8);+ uint w1[16], w0[16];+ for (uint k = 0; k < 16u; k++) {+ w1[k] = p.level >= 2u ? P1.v[pw + k] : 0u;+ w0[k] = p.level >= 3u ? P0.v[pw + k] : 0u;+ }+ for (uint v = 0; v < NV; v++) {+ uvec4 xr[NX];+ for (uint j = 0; j < NX; j++)+ xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];+ int isum = 0;+ for (uint n = 0; n < 2u; n++)+ for (uint part = 0; part < 4u; part++) {+ /* 32 values: 2 groups of 16, each with its scale */+ uint hsh = part * 2u;+ uvec4 a0 = bc(xr, 8u * n + 2u * part);+ uvec4 a1 = bc(xr, 8u * n + 2u * part + 1u);+ uint a[8] = uint[8](a0.x, a0.y, a0.z, a0.w, a1.x, a1.y, a1.z, a1.w);+ int dot0 = 0, dot1 = 0;+ for (uint k = 0; k < 8u; k++) {+ uint i = 8u * n + k;+ uint q = (wd[i] >> hsh) & 0x03030303u;+ if (p.level >= 2u)+ q = (q << 2) | ((w1[i] >> hsh) & 0x03030303u);+ if (p.level >= 3u)+ q = (q << 2) | ((w0[i] >> hsh) & 0x03030303u);+ if (k < 4u)+ dot0 += dp(q, a[k]);+ else+ dot1 += dp(q, a[k]);+ }+ uint g = 8u * n + 2u * part;+ isum += sc[g] * dot0 + sc[g + 1u] * dot1;+ }+ /* the centre of the missing planes, from the block sums */+ uvec4 bs0 = bc(xr, 16u), bs1 = bc(xr, 17u);+ uint bw[8] = uint[8](bs0.x, bs0.y, bs0.z, bs0.w, bs1.x, bs1.y, bs1.z, bs1.w);+ int boff = 0;+ for (uint s = 0; s < 8u; s++)+ boff += sc[2u * s] * bitfieldExtract(int(bw[s]), 0, 16) ++ sc[2u * s + 1u] * bitfieldExtract(int(bw[s]), 16, 16);+ float d8 = uintBitsToFloat(bc(xr, 18u).x);+ acc[v] = fma(d * 0.5 * d8, float(isum * (1 << shk) - c * boff), acc[v]);+ xo[v] += 20u;+ }+ }+ if (gl_GlobalInvocationID.x < p.rows)+ for (uint v = 0; v < NV; v++)+ if (v0 + v < p.nv)+ Y.v[(v0 + v) * p.ys + row] = acc[v];+}diff --git a/src/llm/tuner.c b/src/llm/tuner.cindex 858864b..e84d1d3 100644--- a/src/llm/tuner.c+++ b/src/llm/tuner.c@@ -40,12 +40,12 @@ void janas_tuner_init(struct janas_tuner *t, const struct janas_tune_cand *c,}}-/* GPU candidates: only when allowed, and never for one token (the GPU- takes no part in single-token products: they would differ only by the+/* GPU candidates: only when allowed, and for one token only where the+ GPU takes single-token products (elsewhere they would differ only by thepower its keep-alive costs) */static int usable(const struct janas_tuner *t, int cls, int i){- return (!t->cand[i].gpu || (t->allow_gpu && cls > 0)) &&+ return (!t->cand[i].gpu || (t->allow_gpu && (cls > 0 || t->gpu_one))) &&(!t->cls[cls].threads || t->cand[i].threads == t->cls[cls].threads);}@@ -83,6 +83,11 @@ void janas_tuner_allow_gpu(struct janas_tuner *t, int allow)t->allow_gpu = allow;}+void janas_tuner_allow_gpu_one(struct janas_tuner *t, int allow)+{+ t->gpu_one = allow;+}+int janas_tuner_best(const struct janas_tuner *t, int cls){/* the default unless an alternative is clearly faster: measurementsdiff --git a/src/llm/tuner.h b/src/llm/tuner.hindex 2852a3e..6f0303f 100644--- a/src/llm/tuner.h+++ b/src/llm/tuner.h@@ -26,7 +26,7 @@struct janas_tune_cand {int threads; /* compute threads (a prefix of the pool) */- int gpu; /* the GPU takes part in products over blocks */+ int gpu; /* the GPU takes part in the products */};struct janas_tuner {@@ -34,6 +34,7 @@ struct janas_tuner {int n_cand;int fixed; /* no tuning: the default candidate of each class */int allow_gpu; /* candidates with the GPU may be chosen */+ int gpu_one; /* ... for one token too (the GPU takes single tokens) */int forced; /* >= 0: every pass uses this candidate (and is measured) */struct {double cost[JANAS_TUNE_MAX_CANDS]; /* seconds per token, averaged */@@ -66,6 +67,9 @@ void janas_tuner_force(struct janas_tuner *t, int cand);/* Whether candidates using the GPU may be chosen (energy saving: no). */void janas_tuner_allow_gpu(struct janas_tuner *t, int allow);+/* Whether they may be chosen for one token too: only where the GPU takes+ single-token products (janas_gpu_min_n 1, a discrete GPU). */+void janas_tuner_allow_gpu_one(struct janas_tuner *t, int allow);/* The best known candidate of a class (for work that is not measured). */int janas_tuner_best(const struct janas_tuner *t, int cls);diff --git a/tests/gpu_check.c b/tests/gpu_check.cindex 8569fc5..c300c32 100644--- a/tests/gpu_check.c+++ b/tests/gpu_check.c@@ -2,8 +2,9 @@Copyright (C) 2026 Maurizio Cammalleri *//** gpu_check.c - products split between the GPU and the CPU must equal the- * CPU's bit for bit: random Q4_K matrices in shared memory, several shapes,- * vector counts and strides. Skips (exit 0) when no GPU is usable.+ * CPU's bit for bit: random Q4_K, Q5_K, Q6_K and Q6_K_P (bit planes, at its+ * three levels) matrices in shared memory, several shapes, vector counts and+ * strides. Skips (exit 0) when no GPU is usable.** Usage: gpu_check*/@@ -63,8 +64,34 @@ int main(void)struct janas_block_q6k *w6b = (struct janas_block_q6k *)w6;for (size_t b = 0; b < (w6bytes - 4096) / 210; b++)w6b[b].d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);+ /* Q6_K_P: the high planes (82-byte blocks), then the two bare planes+ (64 bytes a block), row after row as in an expert slot */+ size_t pbase = (size_t)MAXR * (MAXC / JANAS_QK) * 82 + 4096;+ pbase = (pbase + 4095) / 4096 * 4096;+ size_t pplane = (size_t)MAXR * (MAXC / JANAS_QK) * JANAS_Q6KP_PLANE;+ size_t wpbytes = pbase + 2 * pplane;+ uint8_t *wp = aligned_alloc(4096, wpbytes);+ for (size_t i = 0; i < wpbytes; i++)+ wp[i] = (uint8_t)rnd();+ struct janas_block_q6kp *wpb = (struct janas_block_q6kp *)wp;+ for (size_t b = 0; b < (pbase - 4096) / 82; b++)+ wpb[b].d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);+ const uint8_t *pl1 = wp + pbase, *pl0 = wp + pbase + pplane;+ /* Q5_K */+ size_t w5bytes = (size_t)MAXR * (MAXC / JANAS_QK) * 176;+ w5bytes = (w5bytes + 4095) / 4096 * 4096;+ uint8_t *w5 = aligned_alloc(4096, w5bytes);+ for (size_t i = 0; i < w5bytes; i++)+ w5[i] = (uint8_t)rnd();+ struct janas_block_q5k *w5b = (struct janas_block_q5k *)w5;+ for (size_t b = 0; b < w5bytes / 176; b++) {+ w5b[b].d = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);+ w5b[b].dmin = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);+ }if (janas_gpu_import(g, w, wbytes) != 0 ||- janas_gpu_import(g, w6, w6bytes) != 0) {+ janas_gpu_import(g, w6, w6bytes) != 0 ||+ janas_gpu_import(g, wp, wpbytes) != 0 ||+ janas_gpu_import(g, w5, w5bytes) != 0) {printf("gpu_check: import failed\n");return 1;}@@ -91,17 +118,31 @@ int main(void){1500, 768}}; /* odd blocks per row */const size_t nvs[] = {2, 3, 5, 8, 9, 16, 17};int bad = 0, runs = 0;- for (int q6 = 0; q6 < 2; q6++)+ /* 0 Q4_K, 1 Q6_K, 2-4 Q6_K_P with three, two and one planes, 5 Q5_K */+ static const char *names[] = {"Q4_K", "Q6_K", "Q6_K_P/3",+ "Q6_K_P/2", "Q6_K_P/1", "Q5_K"};+ for (int q6 = 0; q6 < 6; q6++)for (size_t s = 0; s < sizeof(shapes) / sizeof(shapes[0]); s++)for (size_t k = 0; k < sizeof(nvs) / sizeof(nvs[0]); k++)for (int strided = 0; strided < 2; strided++) {size_t rows = shapes[s][0], cols = shapes[s][1];size_t nb = cols / JANAS_QK, nv = nvs[k];+ size_t skip = s % 2; /* a row further in, every other+ shape */struct janas_matvec_task t = {- .type = q6 ? JANAS_Q6_K : JANAS_Q4_K,- /* a row further in, every other shape */- .w = q6 ? w6 + (s % 2) * 210 * nb- : w + (s % 2) * 144 * nb,+ .type = q6 == 5 ? JANAS_Q5_K+ : q6 >= 2 ? JANAS_Q6_K_P+ : q6 == 1 ? JANAS_Q6_K+ : JANAS_Q4_K,+ .w = q6 == 5 ? w5 + skip * 176 * nb+ : q6 >= 2 ? wp + skip * 82 * nb+ : q6 == 1 ? w6 + skip * 210 * nb+ : w + skip * 144 * nb,+ .plane = {q6 == 2 || q6 == 3+ ? pl1 + skip * JANAS_Q6KP_PLANE * nb+ : NULL,+ q6 == 2 ? pl0 + skip * JANAS_Q6KP_PLANE * nb+ : NULL},.x = x,.rows = rows,.cols = cols,@@ -150,9 +191,8 @@ int main(void)printf("%s %zux%zu nv %zu strided %d: v %zu row %zu: ""%g vs %g\n",- q6 ? "Q6_K" : "Q4_K", rows, cols, nv,- strided, v, r, ya[v * ys + r],- yb[v * ys + r]);+ names[q6], rows, cols, nv, strided, v, r,+ ya[v * ys + r], yb[v * ys + r]);for (size_t u = 0;getenv("GPU_CHECK_DEBUG") && u < 2 * MAXV;u++)@@ -173,6 +213,9 @@ int main(void)janas_gpu_destroy(g);janas_pool_destroy(pool);free(w);+ free(w6);+ free(wp);+ free(w5);free(x);free(xf);free(ya);diff --git a/tests/test_expert_cache.c b/tests/test_expert_cache.cindex 704f661..719b10e 100644--- a/tests/test_expert_cache.c+++ b/tests/test_expert_cache.c@@ -210,6 +210,104 @@ static int model_fetch(struct model *m, int layer, const uint32_t *ids, int n)return hits;}+/* Every expert of every layer in the arena, nothing being read: only then+ does janas_expert_cache_resident hand the arena out (for a GPU's copy). */+static void check_resident(struct janas_jns *j, struct janas_pool *io)+{+ enum { ALL = N_LAYER * N_EXPERT };+ char err[256];+ uint64_t bytes = 0, loads = 0;+ static uint8_t expect[SLOT];++ /* fewer slots than experts: never */+ struct janas_expert_cache *c = janas_expert_cache_create(+ j, (uint64_t)SLOTS * SLOT, 3, io, err, sizeof(err));+ CHECK(c != NULL, "create (small): %s", err);+ if (c) {+ uint32_t ids[K] = {0, 1, 2};+ const uint8_t *slots[K];+ for (uint32_t l = 0; l < N_LAYER; l++)+ CHECK(janas_expert_cache_fetch(c, l, ids, K, slots) == 0, "fetch");+ CHECK(janas_expert_cache_resident(c, &bytes, &loads) == NULL,+ "resident with %d slots for %d experts", SLOTS, ALL);+ janas_expert_cache_destroy(c);+ }++ /* room for all: not while one is missing, then the whole arena */+ c = janas_expert_cache_create(j, (uint64_t)ALL * SLOT, 3, io, err,+ sizeof(err));+ CHECK(c != NULL, "create (all): %s", err);+ if (c) {+ CHECK(janas_expert_cache_slots(c) == ALL, "slots %zu",+ janas_expert_cache_slots(c));+ const uint8_t *slot;+ for (uint32_t k = 0; k + 1 < ALL; k++) {+ uint32_t e = k % N_EXPERT;+ CHECK(janas_expert_cache_fetch(c, k / N_EXPERT, &e, 1, &slot) == 0,+ "fetch");+ }+ CHECK(janas_expert_cache_resident(c, &bytes, &loads) == NULL,+ "resident with one expert missing");+ /* the last one asked for and still being read: its slot is taken,+ so the arena is full, but its bytes are not there yet */+ uint32_t last = N_EXPERT - 1;+ uint8_t ready;+ CHECK(janas_expert_cache_begin(c, N_LAYER - 1, &last, 1, &slot,+ &ready) == 1,+ "begin did not queue the last expert");+ CHECK(janas_expert_cache_resident(c, &bytes, &loads) == NULL,+ "resident while an expert is being read");+ CHECK(janas_expert_cache_finish(c) == 0, "finish failed");+ const uint8_t *a = janas_expert_cache_resident(c, &bytes, &loads);+ CHECK(a != NULL, "not resident with every expert loaded");+ CHECK(bytes == (uint64_t)ALL * SLOT, "resident bytes %llu",+ (unsigned long long)bytes);+ CHECK(loads == ALL && janas_expert_cache_loads(c) == ALL,+ "loads %llu, expected %d", (unsigned long long)loads, ALL);+ /* every expert is in the arena, where fetch says it is */+ for (uint32_t k = 0; a && k < ALL; k++) {+ uint32_t e = k % N_EXPERT;+ CHECK(janas_expert_cache_fetch(c, k / N_EXPERT, &e, 1, &slot) == 0,+ "fetch");+ CHECK(slot >= a && slot + SLOT <= a + bytes,+ "slot outside the arena");+ fill_slot(expect, k / N_EXPERT, e);+ CHECK(memcmp(slot, expect, SLOT) == 0, "arena: wrong bytes");+ }+ /* hits load nothing: a GPU's copy stays true */+ CHECK(janas_expert_cache_loads(c) == ALL, "hits changed loads to %llu",+ (unsigned long long)janas_expert_cache_loads(c));+ janas_expert_cache_destroy(c);+ }++ /* filled in the background: never while it runs, then resident */+ c = janas_expert_cache_create(j, (uint64_t)ALL * SLOT, 3, io, err,+ sizeof(err));+ uint32_t *counts = calloc(ALL, sizeof(uint32_t));+ if (c && counts) {+ for (size_t k = 0; k < ALL; k++)+ counts[k] = (uint32_t)(k % 5) + 1;+ CHECK(janas_expert_cache_warm_background(c, counts) == 0,+ "background filling did not start");+ /* the first time it answers, the filling is over and complete */+ const uint8_t *a = NULL;+ for (int t = 0; t < 10000 && !a; t++) {+ a = janas_expert_cache_resident(c, &bytes, &loads);+ if (!a)+ usleep(1000);+ }+ uint64_t done = 0, total = 0;+ janas_expert_cache_warm_progress(c, &done, &total);+ CHECK(a != NULL, "not resident 10 s after the filling began");+ CHECK(!a || (loads == ALL && done == total),+ "resident with %llu loads, filling at %llu of %llu",+ (unsigned long long)loads, (unsigned long long)done,+ (unsigned long long)total);+ }+ janas_expert_cache_destroy(c);+ free(counts);+}+/* The whole check, once with the reads on the ring and once without it. */static void run(const char *path, const char *how){@@ -338,6 +436,7 @@ static void run(const char *path, const char *how)}janas_expert_cache_destroy(c);+ check_resident(&j, io);janas_pool_destroy(io);janas_jns_close(&j);printf("test_expert_cache (%s): %llu requests, %llu hits (model %llu)\n",diff --git a/tests/test_tuner.c b/tests/test_tuner.cindex d2ff40f..0fcb105 100644--- a/tests/test_tuner.c+++ b/tests/test_tuner.c@@ -79,6 +79,35 @@ int main(void)}CHECK(!t.cand[janas_tuner_best(&t, 0)].gpu, "GPU best for one token");+ /* ... unless the GPU takes single tokens: then it is tried, and kept+ when faster */+ {+ struct janas_tuner g;+ janas_tuner_init(&g, cand, 4, def, 0);+ janas_tuner_allow_gpu(&g, 1);+ janas_tuner_allow_gpu_one(&g, 1);+ int tried_gpu = 0;+ for (int i = 0; i < 300; i++) {+ int c = janas_tuner_pick(&g, 0);+ tried_gpu += g.cand[c].gpu;+ janas_tuner_record(&g, 0, c, c == 3 ? 0.5 : 1.0);+ }+ CHECK(tried_gpu > 0, "GPU never tried for one token");+ CHECK(g.cand[janas_tuner_best(&g, 0)].gpu,+ "faster GPU not best for one token (best %d)",+ janas_tuner_best(&g, 0));+ /* slower: tried, not kept */+ janas_tuner_init(&g, cand, 4, def, 0);+ janas_tuner_allow_gpu(&g, 1);+ janas_tuner_allow_gpu_one(&g, 1);+ for (int i = 0; i < 300; i++) {+ int c = janas_tuner_pick(&g, 0);+ janas_tuner_record(&g, 0, c, c == 3 ? 2.0 : 1.0);+ }+ CHECK(!g.cand[janas_tuner_best(&g, 0)].gpu,+ "slower GPU best for one token");+ }+/* saved and loaded under a key (in a scratch cache directory) */char dir[512];const char *tmp = getenv("TMPDIR");