37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* expert_cache.h - a RAM cache of routed experts over a JNS file.
*
* A fixed number of slots, each large enough for the largest expert of the
* model, lives in one 4 KiB-aligned arena (an MTP block whose experts are
* larger than the model's has slots of its own: see create_split). Replacement
* is LRU; the cache can be warmed with the experts most used by a prompt (the
* prefill predicts the decode working set: bozza Fase 0). The misses of a
* request are read in parallel with O_DIRECT by a pool of I/O threads, and the
* experts of the request itself are never evicted to make room for each other.
*/
#ifndef JANAS_LLM_EXPERT_CACHE_H
#define JANAS_LLM_EXPERT_CACHE_H
#include <stddef.h>
#include <stdint.h>
#include "common/pool.h"
#include "jns.h"
struct janas_expert_cache_stats {
uint64_t requests; /* experts asked for */
uint64_t hits;
uint64_t misses;
uint64_t bytes_read; /* from the file, O_DIRECT, for the requests */
uint64_t warm_bytes; /* and for the filling in the background */
double io_seconds; /* wall time of the reads (in the background) */
double wait_seconds; /* time the caller actually waited for them */
/* reads ahead (janas_expert_cache_prefetch): experts read, of those
the ones their layer then asked for, and the bytes (not in
bytes_read) */
uint64_t prefetch_reads, prefetch_used, prefetch_bytes;
};
struct janas_expert_cache;
/*
* Creates a cache of budget_bytes (rounded down to whole slots) over an open
* JNS file. io is the pool that performs the reads (its threads should not be
* the compute threads). level, 1 to 3, is how much of each expert slot is
* read: with a file whose down matrix is in bit planes, level 1 reads the
* base plane alone (two bits per weight), 2 adds the second plane (four) and
* 3 the whole slot (six, the Q6_K weights themselves). A smaller level means
* smaller slots, so the same budget holds more experts. Returns NULL on
* failure, with a message in err.
*/
struct janas_expert_cache *janas_expert_cache_create(const struct janas_jns *j,
uint64_t budget_bytes,
int level,
struct janas_pool *io,
char *err, size_t err_len);
/* The most experts of one layer the engine asks for in one request
(model_ffn.c): the fewest slots a part of the cache may have. */
#define JANAS_EXPERT_CHUNK 128
/*
* The same over the file's first n_layer layers (0: all), of which the
* first n_main are the model's own and the others an MTP block in the same
* file (0: all the model's). Where the block's experts are larger than the
* model's - Qwen3.6-35B-A3B's in Q8_0, 3,342,336 bytes against 1,835,008 -
* slots all as large as theirs would hold half the experts the budget can
* (issue #34), so the block gets slots of its own, in the same arena and
* out of the same budget: as many as one request asks for, or all of its
* experts when the budget holds every expert of both. Its slots are whole
* multiples of the model's, so the arena is still cut in slots of the
* model's size wherever a GPU imports it. Only where that holds more
* experts than one size for all; otherwise, and where the block's experts
* are no larger, the cache is the one of janas_expert_cache_create.
* Requests, reads ahead and questions for the block's layers go to its
* slots; the counts of janas_expert_cache_stats and _loads are of both,
* _slots and _slot_bytes the model's, and the filling (warm) is of the
* model's layers alone - the block's few slots fill as it asks.
*/
struct janas_expert_cache *
janas_expert_cache_create_split(const struct janas_jns *j, uint32_t n_layer,
uint32_t n_main, uint64_t budget_bytes,
int level, struct janas_pool *io, char *err,
size_t err_len);
/* The bytes of slots create_split takes to hold every expert of those
layers at that level: the most a budget for them is worth. */
uint64_t janas_expert_cache_room(const struct janas_jns *j, uint32_t n_layer,
uint32_t n_main, int level);
/* The bytes of an expert slot this cache holds, at the level it was made
(of the model's layers, with an MTP block apart). */
uint64_t janas_expert_cache_slot_bytes(const struct janas_expert_cache *c);
void janas_expert_cache_destroy(struct janas_expert_cache *c);
/* Its slots (for the model's layers, with an MTP block apart). */
size_t janas_expert_cache_slots(const struct janas_expert_cache *c);
/*
* The arena, when it holds every expert of every layer and nothing is
* being read into it: from then on no slot changes (there is nothing left
* to load), so a copy of it stays true - for a GPU. NULL otherwise, and
* between begin and finish. loads: the experts ever put in a slot, to tell
* later whether one changed.
*/
const uint8_t *janas_expert_cache_resident(struct janas_expert_cache *c,
uint64_t *bytes, uint64_t *loads);
/* The arena and its bytes, whatever it holds: for a GPU that reads the
host's memory where it is, which sees each slot as it is loaded. The
bytes given here and by _resident count a guard page past the last
slot (JANAS_ARENA_GUARD): the GPU's shaders read a few bytes beyond a
matrix (Q6_K: 4), and the last slot's had nowhere to go. */
#define JANAS_ARENA_GUARD 4096
/* the boundary of the memory a GPU may import: a huge page */
#define JANAS_HUGE_PAGE ((size_t)2 << 20)
const uint8_t *janas_expert_cache_arena(const struct janas_expert_cache *c,
uint64_t *bytes);
uint64_t janas_expert_cache_loads(const struct janas_expert_cache *c);
/*
* The arena moved to arena (the bytes _arena gives, guard included, owned by
* the caller, not freed here), its contents copied, the file read without
* O_DIRECT from then on: for memory a GPU's driver allocated, which it reads
* faster than imported pages. Only when nothing is being read (as _resident);
* 0, or -1 with nothing changed.
*/
int janas_expert_cache_rebase(struct janas_expert_cache *c, uint8_t *arena);
/*
* Loads up to the cache capacity the experts with the highest counts
* (counts has n_layer * n_expert entries, layer-major; zero means never;
* those of an MTP block with slots apart are not read).
* The most frequent end up most recently used. Returns 0 or -1 on I/O error.
*/
int janas_expert_cache_warm(struct janas_expert_cache *c,
const uint32_t *counts);
/*
* The same, on a thread of its own, so that the first replies do not wait
* for it: the most used experts are read first, in batches, and a request
* from the caller is served between two of them. It stops on its own when
* the cache has no empty slot left, so it never evicts anything. counts is
* copied. Returns 0, or -1 if it cannot start.
*/
int janas_expert_cache_warm_background(struct janas_expert_cache *c,
const uint32_t *counts);
/*
* Whether the background filling is taking the disk and the memory right
* now. It is not while it waits for the caller to fall idle - which is most
* of the time, since it only fills when nobody is asking - and a pass
* measured then is as good as any other. Do not read it from "done < total":
* the filling stops as soon as the cache is full, and the experts the caller
* asked for on its own fill slots the filling then never counts, so done
* ends below total and stays there.
*/
int janas_expert_cache_warming(const struct janas_expert_cache *c);
/* Experts loaded by the background filling so far, and how many it means
to load (0 and 0 when none was started). */
void janas_expert_cache_warm_progress(const struct janas_expert_cache *c,
uint64_t *done, uint64_t *total);
/*
* Returns in slots[i] the bytes of expert ids[i] of the layer, reading the
* missing ones. The pointers stay valid until the next call. n must not
* exceed the number of slots. Returns 0 or -1 on I/O error.
*/
int janas_expert_cache_fetch(struct janas_expert_cache *c, uint32_t layer,
const uint32_t *ids, size_t n,
const uint8_t **slots);
/* The same for more than 256 experts (a prompt's block on the GPU asks for
up to all of a layer's), up to half the slots: requests of 256 that count
as one, so that none takes a slot an earlier one gave. 0, or -1 (a slot
not given, or I/O error). */
int janas_expert_cache_fetch_many(struct janas_expert_cache *c, uint32_t layer,
const uint32_t *ids, size_t n,
const uint8_t **slots);
/*
* The same request in two steps, so that computing on the experts already in
* RAM overlaps the reads of the missing ones. begin fills slots[] for every
* expert and sets ready[i] = 1 for those already cached; the reads of the
* others run in the background until janas_expert_cache_finish(), after which
* every slot is valid. Only one request may be outstanding.
* begin returns the number of experts being read, or -1.
*/
int janas_expert_cache_begin(struct janas_expert_cache *c, uint32_t layer,
const uint32_t *ids, size_t n,
const uint8_t **slots, uint8_t *ready);
int janas_expert_cache_finish(struct janas_expert_cache *c);
/*
* Reads ahead up to max of the experts ids of the layer that are not in the
* cache (256 at most, and an eighth of the cache's slots), in the order
* given (the most likely first), while the caller goes
* on: for a layer that will ask soon. They take the least recently used
* slots and stay last in line, and until that layer's next request they are
* never taken back (a read in flight must not land in a slot given to
* another); that request waits for them, counts those it asks for as hits,
* and leaves the others first in line to go. Called between requests, not
* between begin and finish; one layer ahead at a time (a new call for
* another layer settles the one before). Only where the kernel has io_uring,
* on a ring of its own, so that a request never waits for reads ahead of
* another layer. Returns the number of reads started (0: none, or no ring).
*/
int janas_expert_cache_prefetch(struct janas_expert_cache *c, uint32_t layer,
const uint32_t *ids, size_t n, size_t max);
/* Whether this cache can read ahead at all (the kernel has io_uring). */
int janas_expert_cache_can_prefetch(const struct janas_expert_cache *c);
/*
* in[i] = 1 where expert ids[i] of the layer is in the cache (or being read
* into it), 0 where a request would read it. Nothing is touched: a question,
* not a request. Between requests, not between begin and finish.
*/
void janas_expert_cache_present(struct janas_expert_cache *c, uint32_t layer,
const uint32_t *ids, size_t n, uint8_t *in);
void janas_expert_cache_stats(const struct janas_expert_cache *c,
struct janas_expert_cache_stats *s);
void janas_expert_cache_reset_stats(struct janas_expert_cache *c);
#endif