Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

Release 2026-09-28: more of the work on the GPU

camauri97149f48ba7d
diff --git a/ChangeLog.md b/ChangeLog.md
index e45d916..b4052c9 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -2,6 +2,29 @@
> Curated, user-facing summary of completed work, newest first.
+## [2026-09-28] - More of the work on the GPU
+
+- **The experts' weights go to the GPU too** once every expert of every
+ layer is in the cache: the feed-forward weights of a dense model, and a
+ mixture small enough to fit. They were always left to the CPU. On the
+ development laptop's integrated GPU, Qwen3-4B reads a prompt 29% faster
+ (83 to 107 tokens/s) and the chip spends 36% less energy per token
+ (0.49 to 0.31 J, RAPL package counter); replies are unchanged.
+ `JANAS_GPU_EXPERTS=0` keeps them on the CPU. Regions over 4 GB no longer
+ turn the GPU off: they are imported in pieces.
+- **Single tokens on a discrete GPU**, where the tuner finds it faster. The
+ tuner never gave a reply's single token to the GPU; on an RTX 5080 (issue
+ #15) doing so made replies 11-16% faster, even with only attention and
+ output in the card's memory. A discrete GPU now takes them, and the tuner
+ tries it and keeps it only if it measures faster; integrated GPUs are
+ left as they were.
+- **Two more weight types on the GPU**: Q6_K in bit planes (the down
+ matrices of every Qwen mixture and of some layers of the dense models)
+ and Q5_K (Qwen3.5 2B and 9B). Like the others they give the CPU's
+ results bit for bit, so a product still splits between the two.
+- `JANAS_GPU_VERBOSE=2` lists each product shape once with its weight
+ types and whether the GPU takes them.
+
## [2026-09-27] - No Python left
- The README names its author, Maurizio "camauri" Cammalleri, under the
diff --git a/README.md b/README.md
index 1de1b5c..40bf98f 100644
--- a/README.md
+++ b/README.md
@@ -1079,10 +1079,14 @@ and for [a model that will not convert](https://github.com/prabanta-dev/janas/is
1. **CPUs without AVX-VNNI**, and **AMD** — the arithmetic has a path for them
and it is the least exercised of all.
-2. **Discrete GPUs** (NVIDIA, AMD, Intel Arc). The GPU support is written and
- correct, bit for bit, but on an integrated GPU it gains nothing: whether it
- pays on a real card is unknown. **Careful:** a GPU driver reset takes the
- desktop with it. Start with `janas-bench`, not with a long chat.
+2. **Discrete GPUs** (NVIDIA, AMD, Intel Arc). The GPU support is correct,
+ bit for bit, and it pays where it was measured: on the development
+ laptop's integrated GPU a dense model whose weights all fit in memory
+ reads a prompt about 29% faster (Qwen3-4B, 83 to 107 tokens/s), and on an
+ RTX 5080 (issue #15) Qwen3-4B read 13% faster and, with the GPU on single
+ tokens too, replied 11-16% faster. Every other card is unknown.
+ **Careful:** a GPU driver reset takes the desktop with it. Start with
+ `janas-bench`, not with a long chat.
3. **Machines with 16 GB or less.** The engine is supposed to trade quality for
memory on its own — fewer bits per weight, fewer experts per token — and the
thresholds for that were measured on 32 GB.
diff --git a/src/llm/expert_cache.c b/src/llm/expert_cache.c
index acb1a41..6577cb0 100644
--- a/src/llm/expert_cache.c
+++ b/src/llm/expert_cache.c
@@ -74,8 +74,9 @@ struct janas_expert_cache {
/* one request or one batch of the background filling at a time */
pthread_mutex_t api_lock;
- size_t filled; /* slots that hold an expert */
- double last_request; /* when the caller last asked for experts */
+ size_t filled; /* slots that hold an expert */
+ _Atomic uint64_t loads; /* experts ever put in a slot */
+ double last_request; /* when the caller last asked for experts */
/* the background filling of the cache, if it was started */
pthread_t warmer;
int has_warmer, warm_stop;
@@ -383,6 +384,7 @@ static int32_t load(struct janas_expert_cache *c, uint32_t layer, uint32_t e)
} else {
c->filled++;
}
+ atomic_fetch_add(&c->loads, 1);
int64_t k = (int64_t)layer * c->j->h.n_expert + e;
c->key[s] = k;
c->where[k] = s;
@@ -662,6 +664,34 @@ int janas_expert_cache_warm_background(struct janas_expert_cache *c,
return 0;
}
+const uint8_t *janas_expert_cache_resident(struct janas_expert_cache *c,
+ uint64_t *bytes, uint64_t *loads)
+{
+ size_t total = (size_t)c->j->h.n_layer * c->j->h.n_expert;
+ /* between begin and finish the caller holds api_lock: asked there, the
+ answer is no rather than waiting on itself (outstanding is the
+ caller's own) */
+ if (c->n_slots < total || c->outstanding)
+ return NULL;
+ pthread_mutex_lock(&c->api_lock);
+ int busy = c->has_warmer && !atomic_load(&c->warm_over);
+ pthread_mutex_lock(&c->lock);
+ busy |= c->pending;
+ pthread_mutex_unlock(&c->lock);
+ int full = c->filled >= total;
+ *loads = atomic_load(&c->loads);
+ pthread_mutex_unlock(&c->api_lock);
+ if (busy || !full)
+ return NULL;
+ *bytes = (uint64_t)c->n_slots * c->slot_bytes;
+ return c->arena;
+}
+
+uint64_t janas_expert_cache_loads(const struct janas_expert_cache *c)
+{
+ return atomic_load(&((struct janas_expert_cache *)c)->loads);
+}
+
int janas_expert_cache_warming(const struct janas_expert_cache *c)
{
if (!c->has_warmer)
diff --git a/src/llm/expert_cache.h b/src/llm/expert_cache.h
index a1cd022..d7592ee 100644
--- a/src/llm/expert_cache.h
+++ b/src/llm/expert_cache.h
@@ -53,6 +53,17 @@ void janas_expert_cache_destroy(struct janas_expert_cache *c);
size_t janas_expert_cache_slots(const struct janas_expert_cache *c);
+/*
+ * The arena, when it holds every expert of every layer and nothing is
+ * being read into it: from then on no slot changes (there is nothing left
+ * to load), so a copy of it stays true - for a GPU. NULL otherwise, and
+ * between begin and finish. loads: the experts ever put in a slot, to tell
+ * later whether one changed.
+ */
+const uint8_t *janas_expert_cache_resident(struct janas_expert_cache *c,
+ uint64_t *bytes, uint64_t *loads);
+uint64_t janas_expert_cache_loads(const struct janas_expert_cache *c);
+
/*
* Loads up to the cache capacity the experts with the highest counts
* (counts has n_layer * n_expert entries, layer-major; zero means never).
diff --git a/src/llm/gpu.c b/src/llm/gpu.c
index 5313038..524f839 100644
--- a/src/llm/gpu.c
+++ b/src/llm/gpu.c
@@ -24,11 +24,13 @@
#include <unistd.h>
#include <vulkan/vulkan.h>
-#include "q4k_matvec.h" /* janas_spv_q4k_matvec, generated by build.sh */
-#include "q6k_matvec.h" /* janas_spv_q6k_matvec */
-#include "expert_act.h" /* janas_spv_expert_act */
+#include "q4k_matvec.h" /* janas_spv_q4k_matvec, generated by build.sh */
+#include "q6k_matvec.h" /* janas_spv_q6k_matvec */
+#include "q6kp_matvec.h" /* janas_spv_q6kp_matvec */
+#include "q5k_matvec.h" /* janas_spv_q5k_matvec */
+#include "expert_act.h" /* janas_spv_expert_act */
-#define MAX_REGIONS 8
+#define MAX_REGIONS 32
#define MAX_PENDING 64
#define MAX_XCACHE 8
#define X_STAGE_BYTES ((size_t)8 << 20)
@@ -114,10 +116,12 @@ struct pending {
size_t off; /* in the Y stage */
};
-/* The push constants of the shader. */
+/* The push constants of the shaders (level and the bare planes: Q6_K_P
+ only). */
struct push {
uint32_t w[2], x[2], y[2];
- uint32_t rows, nb, nv, xs, ys;
+ uint32_t rows, nb, nv, xs, ys, level;
+ uint32_t p1[2], p0[2];
};
struct janas_gpu {
@@ -131,9 +135,11 @@ struct janas_gpu {
VkPhysicalDevice pd;
VkDevice dev;
VkQueue queue;
- VkShaderModule sm, sm6, sma;
+ VkShaderModule sm, sm6, sm6p, sm5, sma;
VkPipelineLayout pl;
- VkPipeline pipe, pipe6, pipea; /* Q4_K, Q6_K, expert activation */
+ VkPipeline pipe, pipe6, pipe6p, pipe5, pipea; /* Q4_K, Q6_K, Q6_K_P,
+ Q5_K, experts'
+ activation */
VkCommandPool cp;
VkCommandBuffer cb;
VkFence fence;
@@ -156,7 +162,8 @@ struct janas_gpu {
uint32_t calls, waits;
double t_begin, t_cpu, t_wait, t_copy; /* seconds, verbose only */
uint64_t rows_gpu, rows_all;
- uint32_t off; /* calls left on the CPU alone */
+ uint32_t off; /* calls left on the CPU alone */
+ uint32_t seen; /* its tasks printed (JANAS_GPU_VERBOSE=2) */
} share[SHARE_SLOTS];
size_t min_n;
size_t full_n; /* from this many vectors the GPU takes every row */
@@ -582,6 +589,12 @@ struct janas_gpu *janas_gpu_create(char *err, size_t err_len)
(r = make_pipeline(g, janas_spv_q6k_matvec,
sizeof(janas_spv_q6k_matvec), &g->sm6, &g->pipe6)) !=
VK_SUCCESS ||
+ (r = make_pipeline(g, janas_spv_q6kp_matvec,
+ sizeof(janas_spv_q6kp_matvec), &g->sm6p,
+ &g->pipe6p)) != VK_SUCCESS ||
+ (r = make_pipeline(g, janas_spv_q5k_matvec,
+ sizeof(janas_spv_q5k_matvec), &g->sm5, &g->pipe5)) !=
+ VK_SUCCESS ||
(r = make_pipeline(g, janas_spv_expert_act,
sizeof(janas_spv_expert_act), &g->sma, &g->pipea)) !=
VK_SUCCESS) {
@@ -613,12 +626,16 @@ struct janas_gpu *janas_gpu_create(char *err, size_t err_len)
if (g->keep_us > 0 && record_keep_alive(g) == 0 &&
pthread_create(&g->ka, NULL, keep_alive, g) == 0)
g->ka_running = 1;
+ /* single tokens too on a card with memory of its own: there the GPU
+ adds its bandwidth to the RAM's (measured on an RTX 5080, issue #15:
+ replies 11-16% faster with only attention and output in its memory);
+ an integrated GPU shares the RAM's, and lost 6% */
const char *e = getenv("JANAS_GPU_MIN_N");
- g->min_n = e && atoi(e) > 0 ? (size_t)atoi(e) : 2;
+ g->min_n = e && atoi(e) > 0 ? (size_t)atoi(e) : g->discrete ? 1 : 2;
e = getenv("JANAS_GPU_FULL_N");
g->full_n = e && atoi(e) > 0 ? (size_t)atoi(e) : (size_t)-1;
e = getenv("JANAS_GPU_VERBOSE");
- g->verbose = e && atoi(e) > 0;
+ g->verbose = e && atoi(e) > 0 ? atoi(e) : 0;
g->nospin = getenv("JANAS_GPU_NOSPIN") != NULL;
g->block_wait = getenv("JANAS_GPU_BLOCKWAIT") != NULL;
return g;
@@ -682,12 +699,20 @@ void janas_gpu_destroy(struct janas_gpu *g)
g->vkDestroyPipeline(g->dev, g->pipe, NULL);
if (g->pipe6)
g->vkDestroyPipeline(g->dev, g->pipe6, NULL);
+ if (g->pipe6p)
+ g->vkDestroyPipeline(g->dev, g->pipe6p, NULL);
+ if (g->pipe5)
+ g->vkDestroyPipeline(g->dev, g->pipe5, NULL);
if (g->pipea)
g->vkDestroyPipeline(g->dev, g->pipea, NULL);
if (g->sma)
g->vkDestroyShaderModule(g->dev, g->sma, NULL);
if (g->sm6)
g->vkDestroyShaderModule(g->dev, g->sm6, NULL);
+ if (g->sm6p)
+ g->vkDestroyShaderModule(g->dev, g->sm6p, NULL);
+ if (g->sm5)
+ g->vkDestroyShaderModule(g->dev, g->sm5, NULL);
if (g->pl)
g->vkDestroyPipelineLayout(g->dev, g->pl, NULL);
if (g->sm)
@@ -711,6 +736,11 @@ unsigned janas_gpu_subgroup(const struct janas_gpu *g)
return g->sg;
}
+size_t janas_gpu_min_n(const struct janas_gpu *g)
+{
+ return g->min_n;
+}
+
void janas_gpu_keep_awake(struct janas_gpu *g, int on)
{
if (g)
@@ -971,20 +1001,40 @@ static const struct region *find_region(const struct janas_gpu *g,
return NULL;
}
-/* Bytes the shader may read for rows of a task: Q6_K reads up to 2 bytes
- past the last block (whole words). */
+/* Bytes the shader may read for rows of a task: Q6_K and the high plane of
+ Q6_K_P read up to 2 bytes past the last block (whole words). */
static size_t read_bytes(const struct janas_matvec_task *t, size_t rows)
{
size_t nb = t->cols / JANAS_QK;
return rows * nb * janas_qtype_block_size(t->type) +
- (t->type == JANAS_Q6_K ? 4 : 0);
+ (t->type == JANAS_Q6_K || t->type == JANAS_Q6_K_P ? 4 : 0);
+}
+
+/* The bare planes of a Q6_K_P task: level 3 (both), 2 (plane[0]) or 1. */
+static int plane_level(const struct janas_matvec_task *t)
+{
+ return t->plane[0] && t->plane[1] ? 3 : t->plane[0] ? 2 : 1;
+}
+
+/* The region holding rows of a bare plane (4-byte aligned), or NULL. */
+static const struct region *plane_region(const struct janas_gpu *g,
+ const void *p, size_t rows, size_t nb)
+{
+ return (uintptr_t)p % 4 == 0
+ ? find_region(g, p, rows * nb * JANAS_Q6KP_PLANE)
+ : NULL;
}
int janas_gpu_can(const struct janas_gpu *g, const struct janas_matvec_task *t)
{
size_t nb = t->cols / JANAS_QK, nv = t->n_vec ? t->n_vec : 1;
- int typed = (t->type == JANAS_Q4_K && (uintptr_t)t->w % 16 == 0) ||
- (t->type == JANAS_Q6_K && (uintptr_t)t->w % 4 == 0);
+ int typed =
+ (t->type == JANAS_Q4_K && (uintptr_t)t->w % 16 == 0) ||
+ (t->type == JANAS_Q5_K && (uintptr_t)t->w % 16 == 0) ||
+ (t->type == JANAS_Q6_K && (uintptr_t)t->w % 4 == 0) ||
+ (t->type == JANAS_Q6_K_P && (uintptr_t)t->w % 4 == 0 &&
+ (plane_level(t) < 2 || plane_region(g, t->plane[0], t->rows, nb)) &&
+ (plane_level(t) < 3 || plane_region(g, t->plane[1], t->rows, nb)));
return !g->broken && typed && t->cols % JANAS_QK == 0 &&
nv * nb * GPU_BLOCK <= X_STAGE_BYTES &&
t->rows * nv * sizeof(float) <= Y_STAGE_BYTES &&
@@ -1006,6 +1056,8 @@ static void convert_x(uint8_t *dst, const struct janas_block_q8k *x, size_t xs,
}
}
+static void push_addr(uint32_t out[2], VkDeviceAddress a);
+
int janas_gpu_begin(struct janas_gpu *g, const struct janas_matvec_task *tasks,
const size_t *gpu_rows, size_t n)
{
@@ -1030,7 +1082,10 @@ int janas_gpu_begin(struct janas_gpu *g, const struct janas_matvec_task *tasks,
if (gpu_rows[i] == 0)
continue;
const struct janas_matvec_task *t = &tasks[i];
- VkPipeline want = t->type == JANAS_Q6_K ? g->pipe6 : g->pipe;
+ VkPipeline want = t->type == JANAS_Q6_K ? g->pipe6
+ : t->type == JANAS_Q6_K_P ? g->pipe6p
+ : t->type == JANAS_Q5_K ? g->pipe5
+ : g->pipe;
if (want != bound) {
g->vkCmdBindPipeline(g->cb, VK_PIPELINE_BIND_POINT_COMPUTE, want);
bound = want;
@@ -1073,6 +1128,19 @@ int janas_gpu_begin(struct janas_gpu *g, const struct janas_matvec_task *tasks,
.nv = (uint32_t)nv,
.xs = (uint32_t)nb,
.ys = (uint32_t)gpu_rows[i]};
+ if (t->type == JANAS_Q6_K_P) {
+ pc.level = (uint32_t)plane_level(t);
+ for (int q = 0; q < (int)pc.level - 1; q++) {
+ const struct region *pr =
+ plane_region(g, t->plane[q], gpu_rows[i], nb);
+ if (!pr)
+ return -1;
+ push_addr(q == 0 ? pc.p1 : pc.p0,
+ pr->addr +
+ (VkDeviceAddress)((const uint8_t *)t->plane[q] -
+ pr->host));
+ }
+ }
g->vkCmdPushConstants(g->cb, g->pl, VK_SHADER_STAGE_COMPUTE_BIT, 0,
sizeof(pc), &pc);
g->vkCmdDispatch(
@@ -1331,6 +1399,16 @@ void janas_gpu_matvec_group(struct janas_gpu *g, struct janas_pool *pool,
uint64_t key = (uint64_t)n << 48 ^ (uint64_t)tasks[0].rows << 24 ^
(uint64_t)tasks[0].cols << 8 ^ nv;
struct share *sh = find_share(g, key);
+ /* JANAS_GPU_VERBOSE=2: each shape once, with its tasks' types and
+ whether the GPU can take them */
+ if (g->verbose > 1 && sh && sh->calls == 0 && !sh->seen) {
+ sh->seen = 1;
+ fprintf(stderr, "gpu shape %016llx:", (unsigned long long)key);
+ for (size_t i = 0; i < n; i++)
+ fprintf(stderr, " type %d %zux%zu%s", tasks[i].type, tasks[i].rows,
+ tasks[i].cols, janas_gpu_can(g, &tasks[i]) ? "" : " (CPU)");
+ fprintf(stderr, "\n");
+ }
if (sh && sh->off > 0) {
/* the GPU was late even with the smallest share: CPU for a while */
sh->off--;
@@ -1356,6 +1434,10 @@ void janas_gpu_matvec_group(struct janas_gpu *g, struct janas_pool *pool,
cpu[i] = *t;
cpu[i].w =
(const uint8_t *)t->w + r * nb * janas_qtype_block_size(t->type);
+ for (int q = 0; q < 2; q++) /* Q6_K_P: its bare planes follow */
+ if (t->plane[q])
+ cpu[i].plane[q] =
+ (const uint8_t *)t->plane[q] + r * nb * JANAS_Q6KP_PLANE;
cpu[i].y = t->y + r;
cpu[i].rows = t->rows - r;
cpu[i].y_stride = t->y_stride ? t->y_stride : t->rows;
@@ -1514,6 +1596,12 @@ unsigned janas_gpu_subgroup(const struct janas_gpu *g)
return 0;
}
+size_t janas_gpu_min_n(const struct janas_gpu *g)
+{
+ (void)g;
+ return (size_t)-1;
+}
+
void janas_gpu_matvec_group(struct janas_gpu *g, struct janas_pool *pool,
const struct janas_matvec_task *tasks, size_t n)
{
diff --git a/src/llm/gpu.h b/src/llm/gpu.h
index d22a150..24cc9a1 100644
--- a/src/llm/gpu.h
+++ b/src/llm/gpu.h
@@ -87,5 +87,9 @@ void janas_gpu_keep_awake(struct janas_gpu *g, int on);
/* The device's name, and the subgroup size the shaders run with. */
const char *janas_gpu_name(const struct janas_gpu *g);
unsigned janas_gpu_subgroup(const struct janas_gpu *g);
+/* The fewest vectors a product needs to be given to the GPU: 1 on a
+ discrete GPU (single tokens too), 2 on an integrated one;
+ JANAS_GPU_MIN_N sets it. */
+size_t janas_gpu_min_n(const struct janas_gpu *g);
#endif
diff --git a/src/llm/model.c b/src/llm/model.c
index a48f089..fbd4a9f 100644
--- a/src/llm/model.c
+++ b/src/llm/model.c
@@ -353,6 +353,7 @@ int janas_llm_model_forward(struct janas_llm_model *m, const int32_t *tokens,
from them, or a slow start decides the whole session */
if (r == 0 && dt > 0 && !janas_llm_model_settling(m))
janas_tuner_record(&m->tuner, cls, cand, dt / n);
+ janas_m_arena_to_gpu(m);
return r;
}
diff --git a/src/llm/model_ffn.c b/src/llm/model_ffn.c
index 9995ef9..664c7cd 100644
--- a/src/llm/model_ffn.c
+++ b/src/llm/model_ffn.c
@@ -253,14 +253,14 @@ static void run_experts(struct janas_llm_model *m,
}
if (janas_m_exp_tracing())
ex_setup += now() - t_enter;
- EX_T(ex_t1,
- janas_gpu_matvec_group(m->gpu_off || !m->gpu_use ? NULL : m->gpu,
- m->compute, t1, 2 * (size_t)n_runs));
+ EX_T(ex_t1, janas_gpu_matvec_group(
+ m->gpu_off || !m->gpu_use || m->arena_stale ? NULL : m->gpu,
+ m->compute, t1, 2 * (size_t)n_runs));
struct act_job aj = {m, ng};
EX_T(ex_act, janas_pool_run(m->compute, act_worker, &aj));
- EX_T(ex_t2,
- janas_gpu_matvec_group(m->gpu_off || !m->gpu_use ? NULL : m->gpu,
- m->compute, t2, n_runs));
+ EX_T(ex_t2, janas_gpu_matvec_group(
+ m->gpu_off || !m->gpu_use || m->arena_stale ? NULL : m->gpu,
+ m->compute, t2, n_runs));
}
/* The router's choice for token j: softmax, top-k, normalized weights. */
diff --git a/src/llm/model_impl.h b/src/llm/model_impl.h
index 571a745..789e456 100644
--- a/src/llm/model_impl.h
+++ b/src/llm/model_impl.h
@@ -84,7 +84,13 @@ struct janas_llm_model {
struct janas_gpu *gpu; /* optional, for products over blocks */
int resident_gpu; /* the resident region belongs to the GPU */
int gpu_off; /* weights not shared with it: CPU only */
- char gpu_why[160]; /* no GPU: why (janas_llm_model_gpu_why) */
+ /* the experts' arena on the GPU (dense models, and mixtures whose
+ every expert fits the cache): 0 not yet, 1 there, -1 not possible;
+ arena_stale when a slot changed since - the GPU's copy no longer
+ true, the experts' products stay on the CPU */
+ int arena_gpu, arena_stale;
+ uint64_t arena_loads;
+ char gpu_why[160]; /* no GPU: why (janas_llm_model_gpu_why) */
struct janas_expert_cache *cache;
const struct janas_arch *a; /* what this architecture has (arch.h) */
@@ -324,6 +330,9 @@ struct tok_job {
};
void janas_m_init_tuner(struct janas_llm_model *m);
+int janas_m_gpu_import(struct janas_llm_model *m, const uint8_t *p,
+ uint64_t bytes, uint64_t unit);
+void janas_m_arena_to_gpu(struct janas_llm_model *m);
void janas_m_apply_candidate(struct janas_llm_model *m, int cand);
uint32_t *janas_m_profile_read(const struct janas_llm_model *m, size_t *used);
void janas_m_profile_write(const struct janas_llm_model *m);
diff --git a/src/llm/model_load.c b/src/llm/model_load.c
index 0d093d0..8c16264 100644
--- a/src/llm/model_load.c
+++ b/src/llm/model_load.c
@@ -1039,7 +1039,7 @@ struct janas_llm_model *janas_llm_model_load(const char *path,
flushed out of the CPU's caches before the GPU reads it */
if (m->resident_gpu)
janas_gpu_flush(m->gpu, m->resident, m->j.h.resident_bytes);
- else if (janas_gpu_import(m->gpu, m->resident, m->j.h.resident_bytes) !=
+ else if (janas_m_gpu_import(m, m->resident, m->j.h.resident_bytes, 0) !=
0)
m->gpu_off = 1;
if (m->mtp_res && janas_gpu_import(m->gpu, m->mtp_res,
@@ -1226,6 +1226,78 @@ uint64_t janas_llm_model_warm(const struct janas_llm_model *m, double *seconds)
return m->warm_experts;
}
+/* The GPU region of 4 GB at the most: cut at a multiple of unit (an
+ expert slot), or for the resident region (unit 0) where a tensor
+ begins, so that no tensor straddles two and each stays in reach */
+#define GPU_REGION_MAX ((uint64_t)4 << 30)
+
+static uint64_t resident_cut(const struct janas_llm_model *m, uint64_t from)
+{
+ uint64_t base = m->j.h.resident_offset, best = from;
+ for (uint64_t i = 0; i < m->j.h.n_tensor; i++) {
+ uint64_t at = m->j.tensors[i].offset - base;
+ if (at > from && at <= from + GPU_REGION_MAX && at > best)
+ best = at;
+ }
+ return best > from ? best : from + GPU_REGION_MAX;
+}
+
+int janas_m_gpu_import(struct janas_llm_model *m, const uint8_t *p,
+ uint64_t bytes, uint64_t unit)
+{
+ for (uint64_t at = 0; at < bytes;) {
+ uint64_t end = at + GPU_REGION_MAX;
+ if (end >= bytes)
+ end = bytes;
+ else if (unit)
+ end = at + GPU_REGION_MAX / unit * unit;
+ else
+ end = resident_cut(m, at);
+ if (end > bytes)
+ end = bytes;
+ /* the regions are whole pages: the last one's length rounded up
+ (the allocations are) */
+ uint64_t len = (end - at + 4095) & ~(uint64_t)4095;
+ if (janas_gpu_import(m->gpu, p + at, len) != 0)
+ return -1;
+ at = end;
+ }
+ return 0;
+}
+
+/*
+ * The experts on the GPU too: once the cache holds every expert of every
+ * layer and nothing is being read (a dense model's feed-forwards, or a
+ * mixture that fits), its arena is shared with the GPU or copied to its
+ * memory, and the experts' products are split between the two like the
+ * others. Checked after each pass until then; one change of a slot later
+ * (a new level of bits) and the GPU's copy is no longer used.
+ */
+void janas_m_arena_to_gpu(struct janas_llm_model *m)
+{
+ if (!m->gpu || m->gpu_off || !m->cache || m->arena_gpu < 0)
+ return;
+ if (m->arena_gpu == 1) {
+ if (!m->arena_stale &&
+ janas_expert_cache_loads(m->cache) != m->arena_loads)
+ m->arena_stale = 1;
+ return;
+ }
+ uint64_t bytes, loads;
+ const uint8_t *a = janas_expert_cache_resident(m->cache, &bytes, &loads);
+ if (!a)
+ return;
+ uint64_t slot = janas_expert_cache_slot_bytes(m->cache);
+ if (getenv("JANAS_GPU_EXPERTS") && atoi(getenv("JANAS_GPU_EXPERTS")) == 0)
+ m->arena_gpu = -1;
+ else if (janas_m_gpu_import(m, a, bytes, slot) == 0) {
+ m->arena_gpu = 1;
+ m->arena_loads = loads;
+ } else {
+ m->arena_gpu = -1; /* no room for it: the CPU goes on alone */
+ }
+}
+
int janas_llm_model_settling(const struct janas_llm_model *m)
{
return m->cache && janas_expert_cache_warming(m->cache);
diff --git a/src/llm/model_tune.c b/src/llm/model_tune.c
index 4a78445..10a3944 100644
--- a/src/llm/model_tune.c
+++ b/src/llm/model_tune.c
@@ -41,7 +41,9 @@
* JANAS_TUNE_THREADS=0 keeps every core without trying the others;
* JANAS_TUNE=0 turns tuning off; JANAS_BLOCK_THREADS fixes the threads for
* blocks (and turns it off too); JANAS_GPU_ALWAYS=1 makes the GPU variants
- * the defaults for several tokens.
+ * the defaults for several tokens. The GPU variants are tried for one token
+ * too where the GPU takes single tokens (a discrete GPU, or
+ * JANAS_GPU_MIN_N=1): the default stays without it.
*/
void janas_m_init_tuner(struct janas_llm_model *m)
{
@@ -82,14 +84,21 @@ void janas_m_init_tuner(struct janas_llm_model *m)
}
janas_tuner_init(&m->tuner, c, n, def, (tn && atoi(tn) == 0) || bt);
janas_tuner_allow_gpu(&m->tuner, 1);
- /* the saved choices of this machine and model */
+ /* a GPU that takes single tokens (a discrete one) is tried for replies
+ too, and kept only if it measures faster */
+ int one = m->gpu && !m->gpu_off && janas_gpu_min_n(m->gpu) <= 1;
+ janas_tuner_allow_gpu_one(&m->tuner, one);
+ /* the saved choices of this machine and model; "1g" in the key when
+ the GPU may take replies, so that choices saved without it do not
+ stand in for the tries */
char cpu[128], name[128] = "";
janas_cpu_name(cpu, sizeof(cpu));
janas_jns_meta_str(&m->j, "general.name", name, sizeof(name));
- int w = snprintf(m->tune_key, sizeof(m->tune_key), "v2|%s|%d/%d/%d|%s|%s|",
- cpu, m->p_cores, m->p_threads, m->all_threads,
- m->gpu && !m->gpu_off ? janas_gpu_name(m->gpu) : "no GPU",
- name);
+ int w =
+ snprintf(m->tune_key, sizeof(m->tune_key), "v2|%s|%d/%d/%d|%s%s|%s|",
+ cpu, m->p_cores, m->p_threads, m->all_threads,
+ m->gpu && !m->gpu_off ? janas_gpu_name(m->gpu) : "no GPU",
+ one ? " 1g" : "", name);
for (int k = 0; k < n && w > 0 && (size_t)w < sizeof(m->tune_key); k++)
w += snprintf(m->tune_key + w, sizeof(m->tune_key) - (size_t)w, "%d%s,",
c[k].threads, c[k].gpu ? "g" : "");
diff --git a/src/llm/shaders/q5k_matvec.comp b/src/llm/shaders/q5k_matvec.comp
new file mode 100644
index 0000000..db8536d
--- /dev/null
+++ b/src/llm/shaders/q5k_matvec.comp
@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+#version 450
+// Q5_K weights times up to NV Q8_K vectors per workgroup, one row per
+// invocation: Q4_K with a fifth bit (qh: bit s of byte l is the fifth bit
+// of weight 32 s + l), in the same per-block integer-sum scheme as
+// q4k_matvec.comp and janas_q5k_dot_ref, so the results agree with the CPU
+// bit for bit. A block is 176 bytes, 11 uvec4: d, dmin and the scales, qh
+// in two, the low nibbles in eight. Activations shared across the subgroup
+// as in q4k_matvec.comp.
+#extension GL_EXT_integer_dot_product : require
+#extension GL_KHR_shader_subgroup_basic : require
+#extension GL_KHR_shader_subgroup_ballot : require
+#extension GL_EXT_buffer_reference : require
+#extension GL_EXT_buffer_reference_uvec2 : require
+
+layout(local_size_x = 64) in;
+layout(constant_id = 0) const uint NV = 8;
+layout(constant_id = 1) const uint SG = 8;
+const uint NX = (19u + SG - 1u) / SG;
+
+layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
+layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };
+
+// w: the first row; x: vector v, block b at uvec4 (v * xs + b) * 20;
+// y: row r of vector v at y[v * ys + r]
+layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys; } p;
+
+int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
+int dp4(uvec4 a, uvec4 b) { return dp(a.x, b.x) + dp(a.y, b.y) + dp(a.z, b.z) + dp(a.w, b.w); }
+uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }
+
+void main()
+{
+ U4 W = U4(p.w), X = U4(p.x);
+ F Y = F(p.y);
+ uint lane = gl_SubgroupInvocationID;
+ uint row = min(gl_GlobalInvocationID.x, p.rows - 1u);
+ uint v0 = gl_WorkGroupID.y * NV;
+ uint xo[NV];
+ for (uint v = 0; v < NV; v++)
+ xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;
+ float acc[NV], accm[NV];
+ for (uint v = 0; v < NV; v++) { acc[v] = 0.0; accm[v] = 0.0; }
+ uint wo = row * p.nb * 11u;
+ for (uint b = 0; b < p.nb; b++, wo += 11u) {
+ uvec4 hd = W.v[wo];
+ vec2 dd = unpackHalf2x16(hd.x);
+ int sc[8], mn[8];
+ for (uint j = 0; j < 4u; j++) {
+ sc[j] = int(bitfieldExtract(hd.y, int(8u * j), 6));
+ mn[j] = int(bitfieldExtract(hd.z, int(8u * j), 6));
+ sc[j + 4u] = int(bitfieldExtract(hd.w, int(8u * j), 4) |
+ (bitfieldExtract(hd.y, int(8u * j + 6u), 2) << 4));
+ mn[j + 4u] = int(bitfieldExtract(hd.w, int(8u * j + 4u), 4) |
+ (bitfieldExtract(hd.z, int(8u * j + 6u), 2) << 4));
+ }
+ uvec4 qh0 = W.v[wo + 1u], qh1 = W.v[wo + 2u];
+ uvec4 q[8];
+ for (uint j = 0; j < 8u; j++)
+ q[j] = W.v[wo + 3u + j];
+ for (uint v = 0; v < NV; v++) {
+ uvec4 xr[NX];
+ for (uint j = 0; j < NX; j++)
+ xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];
+ int isum = 0;
+ for (uint c = 0; c < 4u; c++) {
+ /* sub-blocks 2c (low nibbles) and 2c + 1 (high), each 32
+ weights: their fifth bits are bits 2c and 2c + 1 of qh */
+ uvec4 q0 = q[2u * c], q1 = q[2u * c + 1u];
+ uint s0 = 2u * c, s1 = 2u * c + 1u;
+ uvec4 lo0 = (q0 & 0x0f0f0f0fu) | (((qh0 >> s0) & 0x01010101u) << 4);
+ uvec4 lo1 = (q1 & 0x0f0f0f0fu) | (((qh1 >> s0) & 0x01010101u) << 4);
+ uvec4 hi0 = ((q0 >> 4) & 0x0f0f0f0fu) | (((qh0 >> s1) & 0x01010101u) << 4);
+ uvec4 hi1 = ((q1 >> 4) & 0x0f0f0f0fu) | (((qh1 >> s1) & 0x01010101u) << 4);
+ isum += (dp4(lo0, bc(xr, 4u * c)) + dp4(lo1, bc(xr, 4u * c + 1u))) * sc[s0] +
+ (dp4(hi0, bc(xr, 4u * c + 2u)) + dp4(hi1, bc(xr, 4u * c + 3u))) * sc[s1];
+ }
+ uvec4 bs0 = bc(xr, 16u), bs1 = bc(xr, 17u);
+ float d8 = uintBitsToFloat(bc(xr, 18u).x);
+ uint bw[8] = uint[8](bs0.x, bs0.y, bs0.z, bs0.w, bs1.x, bs1.y, bs1.z, bs1.w);
+ int imin = 0;
+ for (uint s = 0; s < 8u; s++)
+ imin += mn[s] * (bitfieldExtract(int(bw[s]), 0, 16) + bitfieldExtract(int(bw[s]), 16, 16));
+ acc[v] = fma(dd.x * d8, float(isum), acc[v]);
+ accm[v] = fma(dd.y * d8, float(imin), accm[v]);
+ xo[v] += 20u;
+ }
+ }
+ if (gl_GlobalInvocationID.x < p.rows)
+ for (uint v = 0; v < NV; v++)
+ if (v0 + v < p.nv)
+ Y.v[(v0 + v) * p.ys + row] = acc[v] - accm[v];
+}
diff --git a/src/llm/shaders/q6kp_matvec.comp b/src/llm/shaders/q6kp_matvec.comp
new file mode 100644
index 0000000..dcac1bd
--- /dev/null
+++ b/src/llm/shaders/q6kp_matvec.comp
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later
+ Copyright (C) 2026 Maurizio Cammalleri */
+#version 450
+// Q6_K in bit planes (JANAS_Q6_K_P, quant.h) times up to NV Q8_K vectors,
+// one row per invocation, as janas_q6kp_dot_ref computes it: the high plane
+// (with the scales and d) in the row's 82-byte blocks, the two bare planes
+// in arrays of their own, 64 bytes a block, each packed as Q6_K's qh field.
+// At level 3 (all three planes) the weight is the Q6_K one; at 2 and 1 the
+// centre of the values the missing planes stand for, folded into the block
+// sum as sumi * 2^shk - c * off, the same integers and float operations as
+// the CPU: the results agree bit for bit. Activations shared across the
+// subgroup as in q4k_matvec.comp. The blocks are 82 bytes, so every other
+// one starts 2 bytes off a 4-byte boundary: words are read from the
+// boundary below and realigned, as in q6k_matvec.comp.
+#extension GL_EXT_integer_dot_product : require
+#extension GL_KHR_shader_subgroup_basic : require
+#extension GL_KHR_shader_subgroup_ballot : require
+#extension GL_EXT_buffer_reference : require
+#extension GL_EXT_buffer_reference_uvec2 : require
+
+layout(local_size_x = 64) in;
+layout(constant_id = 0) const uint NV = 8;
+layout(constant_id = 1) const uint SG = 8;
+const uint NX = (19u + SG - 1u) / SG;
+
+layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer U1 { uint v[]; };
+layout(buffer_reference, std430, buffer_reference_align = 16) readonly buffer U4 { uvec4 v[]; };
+layout(buffer_reference, std430, buffer_reference_align = 4) writeonly buffer F { float v[]; };
+
+// level 3, 2 or 1: the planes present; p1 (level >= 2) and p0 (level 3) the
+// bare planes of the first row, nb * 64 bytes a row
+layout(push_constant) uniform P { uvec2 w, x, y; uint rows, nb, nv, xs, ys, level; uvec2 p1, p0; } p;
+
+int dp(uint a, uint b) { return dotPacked4x8EXT(int(a), int(b)); }
+uvec4 bc(uvec4 xr[NX], uint k) { return subgroupBroadcast(xr[k / SG], k % SG); }
+
+void main()
+{
+ U1 W = U1(p.w), P1 = U1(p.p1), P0 = U1(p.p0);
+ U4 X = U4(p.x);
+ F Y = F(p.y);
+ uint lane = gl_SubgroupInvocationID;
+ uint row = min(gl_GlobalInvocationID.x, p.rows - 1u);
+ uint v0 = gl_WorkGroupID.y * NV;
+ uint xo[NV];
+ for (uint v = 0; v < NV; v++)
+ xo[v] = min(v0 + v, p.nv - 1u) * p.xs * 20u;
+ float acc[NV];
+ for (uint v = 0; v < NV; v++)
+ acc[v] = 0.0;
+ uint shk = p.level == 3u ? 1u : p.level == 2u ? 3u : 5u;
+ int c = p.level == 3u ? 64 : p.level == 2u ? 61 : 49;
+ uint off = row * p.nb * 82u; /* bytes */
+ uint pw = row * p.nb * 16u; /* words of the bare planes */
+ for (uint b = 0; b < p.nb; b++, off += 82u, pw += 16u) {
+ /* the block's 20 words (80 bytes) and d, realigned */
+ uint base = off >> 2, sh = (off & 3u) * 8u;
+ uint wd[21];
+ for (uint k = 0; k < 21u; k++)
+ wd[k] = W.v[base + k];
+ if (sh != 0u)
+ for (uint k = 0; k < 20u; k++)
+ wd[k] = (wd[k] >> 16) | (wd[k + 1u] << 16);
+ float d = unpackHalf2x16(sh != 0u ? wd[20] >> 16 : wd[20] & 0xffffu).x;
+ int sc[16];
+ for (uint k = 0; k < 4u; k++)
+ for (uint j = 0; j < 4u; j++)
+ sc[4u * k + j] = bitfieldExtract(int(wd[16u + k]), int(8u * j), 8);
+ uint w1[16], w0[16];
+ for (uint k = 0; k < 16u; k++) {
+ w1[k] = p.level >= 2u ? P1.v[pw + k] : 0u;
+ w0[k] = p.level >= 3u ? P0.v[pw + k] : 0u;
+ }
+ for (uint v = 0; v < NV; v++) {
+ uvec4 xr[NX];
+ for (uint j = 0; j < NX; j++)
+ xr[j] = X.v[xo[v] + min(j * SG + lane, 19u)];
+ int isum = 0;
+ for (uint n = 0; n < 2u; n++)
+ for (uint part = 0; part < 4u; part++) {
+ /* 32 values: 2 groups of 16, each with its scale */
+ uint hsh = part * 2u;
+ uvec4 a0 = bc(xr, 8u * n + 2u * part);
+ uvec4 a1 = bc(xr, 8u * n + 2u * part + 1u);
+ uint a[8] = uint[8](a0.x, a0.y, a0.z, a0.w, a1.x, a1.y, a1.z, a1.w);
+ int dot0 = 0, dot1 = 0;
+ for (uint k = 0; k < 8u; k++) {
+ uint i = 8u * n + k;
+ uint q = (wd[i] >> hsh) & 0x03030303u;
+ if (p.level >= 2u)
+ q = (q << 2) | ((w1[i] >> hsh) & 0x03030303u);
+ if (p.level >= 3u)
+ q = (q << 2) | ((w0[i] >> hsh) & 0x03030303u);
+ if (k < 4u)
+ dot0 += dp(q, a[k]);
+ else
+ dot1 += dp(q, a[k]);
+ }
+ uint g = 8u * n + 2u * part;
+ isum += sc[g] * dot0 + sc[g + 1u] * dot1;
+ }
+ /* the centre of the missing planes, from the block sums */
+ uvec4 bs0 = bc(xr, 16u), bs1 = bc(xr, 17u);
+ uint bw[8] = uint[8](bs0.x, bs0.y, bs0.z, bs0.w, bs1.x, bs1.y, bs1.z, bs1.w);
+ int boff = 0;
+ for (uint s = 0; s < 8u; s++)
+ boff += sc[2u * s] * bitfieldExtract(int(bw[s]), 0, 16) +
+ sc[2u * s + 1u] * bitfieldExtract(int(bw[s]), 16, 16);
+ float d8 = uintBitsToFloat(bc(xr, 18u).x);
+ acc[v] = fma(d * 0.5 * d8, float(isum * (1 << shk) - c * boff), acc[v]);
+ xo[v] += 20u;
+ }
+ }
+ if (gl_GlobalInvocationID.x < p.rows)
+ for (uint v = 0; v < NV; v++)
+ if (v0 + v < p.nv)
+ Y.v[(v0 + v) * p.ys + row] = acc[v];
+}
diff --git a/src/llm/tuner.c b/src/llm/tuner.c
index 858864b..e84d1d3 100644
--- a/src/llm/tuner.c
+++ b/src/llm/tuner.c
@@ -40,12 +40,12 @@ void janas_tuner_init(struct janas_tuner *t, const struct janas_tune_cand *c,
}
}
-/* GPU candidates: only when allowed, and never for one token (the GPU
- takes no part in single-token products: they would differ only by the
+/* GPU candidates: only when allowed, and for one token only where the
+ GPU takes single-token products (elsewhere they would differ only by the
power its keep-alive costs) */
static int usable(const struct janas_tuner *t, int cls, int i)
{
- return (!t->cand[i].gpu || (t->allow_gpu && cls > 0)) &&
+ return (!t->cand[i].gpu || (t->allow_gpu && (cls > 0 || t->gpu_one))) &&
(!t->cls[cls].threads || t->cand[i].threads == t->cls[cls].threads);
}
@@ -83,6 +83,11 @@ void janas_tuner_allow_gpu(struct janas_tuner *t, int allow)
t->allow_gpu = allow;
}
+void janas_tuner_allow_gpu_one(struct janas_tuner *t, int allow)
+{
+ t->gpu_one = allow;
+}
+
int janas_tuner_best(const struct janas_tuner *t, int cls)
{
/* the default unless an alternative is clearly faster: measurements
diff --git a/src/llm/tuner.h b/src/llm/tuner.h
index 2852a3e..6f0303f 100644
--- a/src/llm/tuner.h
+++ b/src/llm/tuner.h
@@ -26,7 +26,7 @@
struct janas_tune_cand {
int threads; /* compute threads (a prefix of the pool) */
- int gpu; /* the GPU takes part in products over blocks */
+ int gpu; /* the GPU takes part in the products */
};
struct janas_tuner {
@@ -34,6 +34,7 @@ struct janas_tuner {
int n_cand;
int fixed; /* no tuning: the default candidate of each class */
int allow_gpu; /* candidates with the GPU may be chosen */
+ int gpu_one; /* ... for one token too (the GPU takes single tokens) */
int forced; /* >= 0: every pass uses this candidate (and is measured) */
struct {
double cost[JANAS_TUNE_MAX_CANDS]; /* seconds per token, averaged */
@@ -66,6 +67,9 @@ void janas_tuner_force(struct janas_tuner *t, int cand);
/* Whether candidates using the GPU may be chosen (energy saving: no). */
void janas_tuner_allow_gpu(struct janas_tuner *t, int allow);
+/* Whether they may be chosen for one token too: only where the GPU takes
+ single-token products (janas_gpu_min_n 1, a discrete GPU). */
+void janas_tuner_allow_gpu_one(struct janas_tuner *t, int allow);
/* The best known candidate of a class (for work that is not measured). */
int janas_tuner_best(const struct janas_tuner *t, int cls);
diff --git a/tests/gpu_check.c b/tests/gpu_check.c
index 8569fc5..c300c32 100644
--- a/tests/gpu_check.c
+++ b/tests/gpu_check.c
@@ -2,8 +2,9 @@
Copyright (C) 2026 Maurizio Cammalleri */
/*
* gpu_check.c - products split between the GPU and the CPU must equal the
- * CPU's bit for bit: random Q4_K matrices in shared memory, several shapes,
- * vector counts and strides. Skips (exit 0) when no GPU is usable.
+ * CPU's bit for bit: random Q4_K, Q5_K, Q6_K and Q6_K_P (bit planes, at its
+ * three levels) matrices in shared memory, several shapes, vector counts and
+ * strides. Skips (exit 0) when no GPU is usable.
*
* Usage: gpu_check
*/
@@ -63,8 +64,34 @@ int main(void)
struct janas_block_q6k *w6b = (struct janas_block_q6k *)w6;
for (size_t b = 0; b < (w6bytes - 4096) / 210; b++)
w6b[b].d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);
+ /* Q6_K_P: the high planes (82-byte blocks), then the two bare planes
+ (64 bytes a block), row after row as in an expert slot */
+ size_t pbase = (size_t)MAXR * (MAXC / JANAS_QK) * 82 + 4096;
+ pbase = (pbase + 4095) / 4096 * 4096;
+ size_t pplane = (size_t)MAXR * (MAXC / JANAS_QK) * JANAS_Q6KP_PLANE;
+ size_t wpbytes = pbase + 2 * pplane;
+ uint8_t *wp = aligned_alloc(4096, wpbytes);
+ for (size_t i = 0; i < wpbytes; i++)
+ wp[i] = (uint8_t)rnd();
+ struct janas_block_q6kp *wpb = (struct janas_block_q6kp *)wp;
+ for (size_t b = 0; b < (pbase - 4096) / 82; b++)
+ wpb[b].d = janas_fp32_to_fp16(0.0001f + (rnd() % 1000) * 1e-6f);
+ const uint8_t *pl1 = wp + pbase, *pl0 = wp + pbase + pplane;
+ /* Q5_K */
+ size_t w5bytes = (size_t)MAXR * (MAXC / JANAS_QK) * 176;
+ w5bytes = (w5bytes + 4095) / 4096 * 4096;
+ uint8_t *w5 = aligned_alloc(4096, w5bytes);
+ for (size_t i = 0; i < w5bytes; i++)
+ w5[i] = (uint8_t)rnd();
+ struct janas_block_q5k *w5b = (struct janas_block_q5k *)w5;
+ for (size_t b = 0; b < w5bytes / 176; b++) {
+ w5b[b].d = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);
+ w5b[b].dmin = janas_fp32_to_fp16(0.001f + (rnd() % 1000) * 1e-5f);
+ }
if (janas_gpu_import(g, w, wbytes) != 0 ||
- janas_gpu_import(g, w6, w6bytes) != 0) {
+ janas_gpu_import(g, w6, w6bytes) != 0 ||
+ janas_gpu_import(g, wp, wpbytes) != 0 ||
+ janas_gpu_import(g, w5, w5bytes) != 0) {
printf("gpu_check: import failed\n");
return 1;
}
@@ -91,17 +118,31 @@ int main(void)
{1500, 768}}; /* odd blocks per row */
const size_t nvs[] = {2, 3, 5, 8, 9, 16, 17};
int bad = 0, runs = 0;
- for (int q6 = 0; q6 < 2; q6++)
+ /* 0 Q4_K, 1 Q6_K, 2-4 Q6_K_P with three, two and one planes, 5 Q5_K */
+ static const char *names[] = {"Q4_K", "Q6_K", "Q6_K_P/3",
+ "Q6_K_P/2", "Q6_K_P/1", "Q5_K"};
+ for (int q6 = 0; q6 < 6; q6++)
for (size_t s = 0; s < sizeof(shapes) / sizeof(shapes[0]); s++)
for (size_t k = 0; k < sizeof(nvs) / sizeof(nvs[0]); k++)
for (int strided = 0; strided < 2; strided++) {
size_t rows = shapes[s][0], cols = shapes[s][1];
size_t nb = cols / JANAS_QK, nv = nvs[k];
+ size_t skip = s % 2; /* a row further in, every other
+ shape */
struct janas_matvec_task t = {
- .type = q6 ? JANAS_Q6_K : JANAS_Q4_K,
- /* a row further in, every other shape */
- .w = q6 ? w6 + (s % 2) * 210 * nb
- : w + (s % 2) * 144 * nb,
+ .type = q6 == 5 ? JANAS_Q5_K
+ : q6 >= 2 ? JANAS_Q6_K_P
+ : q6 == 1 ? JANAS_Q6_K
+ : JANAS_Q4_K,
+ .w = q6 == 5 ? w5 + skip * 176 * nb
+ : q6 >= 2 ? wp + skip * 82 * nb
+ : q6 == 1 ? w6 + skip * 210 * nb
+ : w + skip * 144 * nb,
+ .plane = {q6 == 2 || q6 == 3
+ ? pl1 + skip * JANAS_Q6KP_PLANE * nb
+ : NULL,
+ q6 == 2 ? pl0 + skip * JANAS_Q6KP_PLANE * nb
+ : NULL},
.x = x,
.rows = rows,
.cols = cols,
@@ -150,9 +191,8 @@ int main(void)
printf(
"%s %zux%zu nv %zu strided %d: v %zu row %zu: "
"%g vs %g\n",
- q6 ? "Q6_K" : "Q4_K", rows, cols, nv,
- strided, v, r, ya[v * ys + r],
- yb[v * ys + r]);
+ names[q6], rows, cols, nv, strided, v, r,
+ ya[v * ys + r], yb[v * ys + r]);
for (size_t u = 0;
getenv("GPU_CHECK_DEBUG") && u < 2 * MAXV;
u++)
@@ -173,6 +213,9 @@ int main(void)
janas_gpu_destroy(g);
janas_pool_destroy(pool);
free(w);
+ free(w6);
+ free(wp);
+ free(w5);
free(x);
free(xf);
free(ya);
diff --git a/tests/test_expert_cache.c b/tests/test_expert_cache.c
index 704f661..719b10e 100644
--- a/tests/test_expert_cache.c
+++ b/tests/test_expert_cache.c
@@ -210,6 +210,104 @@ static int model_fetch(struct model *m, int layer, const uint32_t *ids, int n)
return hits;
}
+/* Every expert of every layer in the arena, nothing being read: only then
+ does janas_expert_cache_resident hand the arena out (for a GPU's copy). */
+static void check_resident(struct janas_jns *j, struct janas_pool *io)
+{
+ enum { ALL = N_LAYER * N_EXPERT };
+ char err[256];
+ uint64_t bytes = 0, loads = 0;
+ static uint8_t expect[SLOT];
+
+ /* fewer slots than experts: never */
+ struct janas_expert_cache *c = janas_expert_cache_create(
+ j, (uint64_t)SLOTS * SLOT, 3, io, err, sizeof(err));
+ CHECK(c != NULL, "create (small): %s", err);
+ if (c) {
+ uint32_t ids[K] = {0, 1, 2};
+ const uint8_t *slots[K];
+ for (uint32_t l = 0; l < N_LAYER; l++)
+ CHECK(janas_expert_cache_fetch(c, l, ids, K, slots) == 0, "fetch");
+ CHECK(janas_expert_cache_resident(c, &bytes, &loads) == NULL,
+ "resident with %d slots for %d experts", SLOTS, ALL);
+ janas_expert_cache_destroy(c);
+ }
+
+ /* room for all: not while one is missing, then the whole arena */
+ c = janas_expert_cache_create(j, (uint64_t)ALL * SLOT, 3, io, err,
+ sizeof(err));
+ CHECK(c != NULL, "create (all): %s", err);
+ if (c) {
+ CHECK(janas_expert_cache_slots(c) == ALL, "slots %zu",
+ janas_expert_cache_slots(c));
+ const uint8_t *slot;
+ for (uint32_t k = 0; k + 1 < ALL; k++) {
+ uint32_t e = k % N_EXPERT;
+ CHECK(janas_expert_cache_fetch(c, k / N_EXPERT, &e, 1, &slot) == 0,
+ "fetch");
+ }
+ CHECK(janas_expert_cache_resident(c, &bytes, &loads) == NULL,
+ "resident with one expert missing");
+ /* the last one asked for and still being read: its slot is taken,
+ so the arena is full, but its bytes are not there yet */
+ uint32_t last = N_EXPERT - 1;
+ uint8_t ready;
+ CHECK(janas_expert_cache_begin(c, N_LAYER - 1, &last, 1, &slot,
+ &ready) == 1,
+ "begin did not queue the last expert");
+ CHECK(janas_expert_cache_resident(c, &bytes, &loads) == NULL,
+ "resident while an expert is being read");
+ CHECK(janas_expert_cache_finish(c) == 0, "finish failed");
+ const uint8_t *a = janas_expert_cache_resident(c, &bytes, &loads);
+ CHECK(a != NULL, "not resident with every expert loaded");
+ CHECK(bytes == (uint64_t)ALL * SLOT, "resident bytes %llu",
+ (unsigned long long)bytes);
+ CHECK(loads == ALL && janas_expert_cache_loads(c) == ALL,
+ "loads %llu, expected %d", (unsigned long long)loads, ALL);
+ /* every expert is in the arena, where fetch says it is */
+ for (uint32_t k = 0; a && k < ALL; k++) {
+ uint32_t e = k % N_EXPERT;
+ CHECK(janas_expert_cache_fetch(c, k / N_EXPERT, &e, 1, &slot) == 0,
+ "fetch");
+ CHECK(slot >= a && slot + SLOT <= a + bytes,
+ "slot outside the arena");
+ fill_slot(expect, k / N_EXPERT, e);
+ CHECK(memcmp(slot, expect, SLOT) == 0, "arena: wrong bytes");
+ }
+ /* hits load nothing: a GPU's copy stays true */
+ CHECK(janas_expert_cache_loads(c) == ALL, "hits changed loads to %llu",
+ (unsigned long long)janas_expert_cache_loads(c));
+ janas_expert_cache_destroy(c);
+ }
+
+ /* filled in the background: never while it runs, then resident */
+ c = janas_expert_cache_create(j, (uint64_t)ALL * SLOT, 3, io, err,
+ sizeof(err));
+ uint32_t *counts = calloc(ALL, sizeof(uint32_t));
+ if (c && counts) {
+ for (size_t k = 0; k < ALL; k++)
+ counts[k] = (uint32_t)(k % 5) + 1;
+ CHECK(janas_expert_cache_warm_background(c, counts) == 0,
+ "background filling did not start");
+ /* the first time it answers, the filling is over and complete */
+ const uint8_t *a = NULL;
+ for (int t = 0; t < 10000 && !a; t++) {
+ a = janas_expert_cache_resident(c, &bytes, &loads);
+ if (!a)
+ usleep(1000);
+ }
+ uint64_t done = 0, total = 0;
+ janas_expert_cache_warm_progress(c, &done, &total);
+ CHECK(a != NULL, "not resident 10 s after the filling began");
+ CHECK(!a || (loads == ALL && done == total),
+ "resident with %llu loads, filling at %llu of %llu",
+ (unsigned long long)loads, (unsigned long long)done,
+ (unsigned long long)total);
+ }
+ janas_expert_cache_destroy(c);
+ free(counts);
+}
+
/* The whole check, once with the reads on the ring and once without it. */
static void run(const char *path, const char *how)
{
@@ -338,6 +436,7 @@ static void run(const char *path, const char *how)
}
janas_expert_cache_destroy(c);
+ check_resident(&j, io);
janas_pool_destroy(io);
janas_jns_close(&j);
printf("test_expert_cache (%s): %llu requests, %llu hits (model %llu)\n",
diff --git a/tests/test_tuner.c b/tests/test_tuner.c
index d2ff40f..0fcb105 100644
--- a/tests/test_tuner.c
+++ b/tests/test_tuner.c
@@ -79,6 +79,35 @@ int main(void)
}
CHECK(!t.cand[janas_tuner_best(&t, 0)].gpu, "GPU best for one token");
+ /* ... unless the GPU takes single tokens: then it is tried, and kept
+ when faster */
+ {
+ struct janas_tuner g;
+ janas_tuner_init(&g, cand, 4, def, 0);
+ janas_tuner_allow_gpu(&g, 1);
+ janas_tuner_allow_gpu_one(&g, 1);
+ int tried_gpu = 0;
+ for (int i = 0; i < 300; i++) {
+ int c = janas_tuner_pick(&g, 0);
+ tried_gpu += g.cand[c].gpu;
+ janas_tuner_record(&g, 0, c, c == 3 ? 0.5 : 1.0);
+ }
+ CHECK(tried_gpu > 0, "GPU never tried for one token");
+ CHECK(g.cand[janas_tuner_best(&g, 0)].gpu,
+ "faster GPU not best for one token (best %d)",
+ janas_tuner_best(&g, 0));
+ /* slower: tried, not kept */
+ janas_tuner_init(&g, cand, 4, def, 0);
+ janas_tuner_allow_gpu(&g, 1);
+ janas_tuner_allow_gpu_one(&g, 1);
+ for (int i = 0; i < 300; i++) {
+ int c = janas_tuner_pick(&g, 0);
+ janas_tuner_record(&g, 0, c, c == 3 ? 2.0 : 1.0);
+ }
+ CHECK(!g.cand[janas_tuner_best(&g, 0)].gpu,
+ "slower GPU best for one token");
+ }
+
/* saved and loaded under a key (in a scratch cache directory) */
char dir[512];
const char *tmp = getenv("TMPDIR");