Skip to content

armgit / janas

Public Imported from GitHub
0

37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.

11.2 KB
Raw Blame
/* SPDX-License-Identifier: GPL-3.0-or-later
   Copyright (C) 2026 Maurizio Cammalleri */
/*
 * expert_cache.h - a RAM cache of routed experts over a JNS file.
 *
 * A fixed number of slots, each large enough for the largest expert of the
 * model, lives in one 4 KiB-aligned arena (an MTP block whose experts are
 * larger than the model's has slots of its own: see create_split). Replacement
 * is LRU; the cache can be warmed with the experts most used by a prompt (the
 * prefill predicts the decode working set: bozza Fase 0). The misses of a
 * request are read in parallel with O_DIRECT by a pool of I/O threads, and the
 * experts of the request itself are never evicted to make room for each other.
 */
#ifndef JANAS_LLM_EXPERT_CACHE_H
#define JANAS_LLM_EXPERT_CACHE_H

#include <stddef.h>
#include <stdint.h>

#include "common/pool.h"
#include "jns.h"

struct janas_expert_cache_stats {
    uint64_t requests; /* experts asked for */
    uint64_t hits;
    uint64_t misses;
    uint64_t bytes_read; /* from the file, O_DIRECT, for the requests */
    uint64_t warm_bytes; /* and for the filling in the background */
    double io_seconds;   /* wall time of the reads (in the background) */
    double wait_seconds; /* time the caller actually waited for them */
    /* reads ahead (janas_expert_cache_prefetch): experts read, of those
       the ones their layer then asked for, and the bytes (not in
       bytes_read) */
    uint64_t prefetch_reads, prefetch_used, prefetch_bytes;
};

struct janas_expert_cache;

/*
 * Creates a cache of budget_bytes (rounded down to whole slots) over an open
 * JNS file. io is the pool that performs the reads (its threads should not be
 * the compute threads). level, 1 to 3, is how much of each expert slot is
 * read: with a file whose down matrix is in bit planes, level 1 reads the
 * base plane alone (two bits per weight), 2 adds the second plane (four) and
 * 3 the whole slot (six, the Q6_K weights themselves). A smaller level means
 * smaller slots, so the same budget holds more experts. Returns NULL on
 * failure, with a message in err.
 */
struct janas_expert_cache *janas_expert_cache_create(const struct janas_jns *j,
                                                     uint64_t budget_bytes,
                                                     int level,
                                                     struct janas_pool *io,
                                                     char *err, size_t err_len);

/* The most experts of one layer the engine asks for in one request
   (model_ffn.c): the fewest slots a part of the cache may have. */
#define JANAS_EXPERT_CHUNK 128

/*
 * The same over the file's first n_layer layers (0: all), of which the
 * first n_main are the model's own and the others an MTP block in the same
 * file (0: all the model's). Where the block's experts are larger than the
 * model's - Qwen3.6-35B-A3B's in Q8_0, 3,342,336 bytes against 1,835,008 -
 * slots all as large as theirs would hold half the experts the budget can
 * (issue #34), so the block gets slots of its own, in the same arena and
 * out of the same budget: as many as one request asks for, or all of its
 * experts when the budget holds every expert of both. Its slots are whole
 * multiples of the model's, so the arena is still cut in slots of the
 * model's size wherever a GPU imports it. Only where that holds more
 * experts than one size for all; otherwise, and where the block's experts
 * are no larger, the cache is the one of janas_expert_cache_create.
 * Requests, reads ahead and questions for the block's layers go to its
 * slots; the counts of janas_expert_cache_stats and _loads are of both,
 * _slots and _slot_bytes the model's, and the filling (warm) is of the
 * model's layers alone - the block's few slots fill as it asks.
 */
struct janas_expert_cache *
janas_expert_cache_create_split(const struct janas_jns *j, uint32_t n_layer,
                                uint32_t n_main, uint64_t budget_bytes,
                                int level, struct janas_pool *io, char *err,
                                size_t err_len);

/* The bytes of slots create_split takes to hold every expert of those
   layers at that level: the most a budget for them is worth. */
uint64_t janas_expert_cache_room(const struct janas_jns *j, uint32_t n_layer,
                                 uint32_t n_main, int level);

/* The bytes of an expert slot this cache holds, at the level it was made
   (of the model's layers, with an MTP block apart). */
uint64_t janas_expert_cache_slot_bytes(const struct janas_expert_cache *c);
void janas_expert_cache_destroy(struct janas_expert_cache *c);

/* Its slots (for the model's layers, with an MTP block apart). */
size_t janas_expert_cache_slots(const struct janas_expert_cache *c);

/*
 * The arena, when it holds every expert of every layer and nothing is
 * being read into it: from then on no slot changes (there is nothing left
 * to load), so a copy of it stays true - for a GPU. NULL otherwise, and
 * between begin and finish. loads: the experts ever put in a slot, to tell
 * later whether one changed.
 */
const uint8_t *janas_expert_cache_resident(struct janas_expert_cache *c,
                                           uint64_t *bytes, uint64_t *loads);
/* The arena and its bytes, whatever it holds: for a GPU that reads the
   host's memory where it is, which sees each slot as it is loaded. The
   bytes given here and by _resident count a guard page past the last
   slot (JANAS_ARENA_GUARD): the GPU's shaders read a few bytes beyond a
   matrix (Q6_K: 4), and the last slot's had nowhere to go. */
#define JANAS_ARENA_GUARD 4096
/* the boundary of the memory a GPU may import: a huge page */
#define JANAS_HUGE_PAGE ((size_t)2 << 20)
const uint8_t *janas_expert_cache_arena(const struct janas_expert_cache *c,
                                        uint64_t *bytes);
uint64_t janas_expert_cache_loads(const struct janas_expert_cache *c);
/*
 * The arena moved to arena (the bytes _arena gives, guard included, owned by
 * the caller, not freed here), its contents copied, the file read without
 * O_DIRECT from then on: for memory a GPU's driver allocated, which it reads
 * faster than imported pages. Only when nothing is being read (as _resident);
 * 0, or -1 with nothing changed.
 */
int janas_expert_cache_rebase(struct janas_expert_cache *c, uint8_t *arena);

/*
 * Loads up to the cache capacity the experts with the highest counts
 * (counts has n_layer * n_expert entries, layer-major; zero means never;
 * those of an MTP block with slots apart are not read).
 * The most frequent end up most recently used. Returns 0 or -1 on I/O error.
 */
int janas_expert_cache_warm(struct janas_expert_cache *c,
                            const uint32_t *counts);

/*
 * The same, on a thread of its own, so that the first replies do not wait
 * for it: the most used experts are read first, in batches, and a request
 * from the caller is served between two of them. It stops on its own when
 * the cache has no empty slot left, so it never evicts anything. counts is
 * copied. Returns 0, or -1 if it cannot start.
 */
int janas_expert_cache_warm_background(struct janas_expert_cache *c,
                                       const uint32_t *counts);

/*
 * Whether the background filling is taking the disk and the memory right
 * now. It is not while it waits for the caller to fall idle - which is most
 * of the time, since it only fills when nobody is asking - and a pass
 * measured then is as good as any other. Do not read it from "done < total":
 * the filling stops as soon as the cache is full, and the experts the caller
 * asked for on its own fill slots the filling then never counts, so done
 * ends below total and stays there.
 */
int janas_expert_cache_warming(const struct janas_expert_cache *c);

/* Experts loaded by the background filling so far, and how many it means
   to load (0 and 0 when none was started). */
void janas_expert_cache_warm_progress(const struct janas_expert_cache *c,
                                      uint64_t *done, uint64_t *total);

/*
 * Returns in slots[i] the bytes of expert ids[i] of the layer, reading the
 * missing ones. The pointers stay valid until the next call. n must not
 * exceed the number of slots. Returns 0 or -1 on I/O error.
 */
int janas_expert_cache_fetch(struct janas_expert_cache *c, uint32_t layer,
                             const uint32_t *ids, size_t n,
                             const uint8_t **slots);
/* The same for more than 256 experts (a prompt's block on the GPU asks for
   up to all of a layer's), up to half the slots: requests of 256 that count
   as one, so that none takes a slot an earlier one gave. 0, or -1 (a slot
   not given, or I/O error). */
int janas_expert_cache_fetch_many(struct janas_expert_cache *c, uint32_t layer,
                                  const uint32_t *ids, size_t n,
                                  const uint8_t **slots);

/*
 * The same request in two steps, so that computing on the experts already in
 * RAM overlaps the reads of the missing ones. begin fills slots[] for every
 * expert and sets ready[i] = 1 for those already cached; the reads of the
 * others run in the background until janas_expert_cache_finish(), after which
 * every slot is valid. Only one request may be outstanding.
 * begin returns the number of experts being read, or -1.
 */
int janas_expert_cache_begin(struct janas_expert_cache *c, uint32_t layer,
                             const uint32_t *ids, size_t n,
                             const uint8_t **slots, uint8_t *ready);
int janas_expert_cache_finish(struct janas_expert_cache *c);

/*
 * Reads ahead up to max of the experts ids of the layer that are not in the
 * cache (256 at most, and an eighth of the cache's slots), in the order
 * given (the most likely first), while the caller goes
 * on: for a layer that will ask soon. They take the least recently used
 * slots and stay last in line, and until that layer's next request they are
 * never taken back (a read in flight must not land in a slot given to
 * another); that request waits for them, counts those it asks for as hits,
 * and leaves the others first in line to go. Called between requests, not
 * between begin and finish; one layer ahead at a time (a new call for
 * another layer settles the one before). Only where the kernel has io_uring,
 * on a ring of its own, so that a request never waits for reads ahead of
 * another layer. Returns the number of reads started (0: none, or no ring).
 */
int janas_expert_cache_prefetch(struct janas_expert_cache *c, uint32_t layer,
                                const uint32_t *ids, size_t n, size_t max);
/* Whether this cache can read ahead at all (the kernel has io_uring). */
int janas_expert_cache_can_prefetch(const struct janas_expert_cache *c);

/*
 * in[i] = 1 where expert ids[i] of the layer is in the cache (or being read
 * into it), 0 where a request would read it. Nothing is touched: a question,
 * not a request. Between requests, not between begin and finish.
 */
void janas_expert_cache_present(struct janas_expert_cache *c, uint32_t layer,
                                const uint32_t *ids, size_t n, uint8_t *in);

void janas_expert_cache_stats(const struct janas_expert_cache *c,
                              struct janas_expert_cache_stats *s);
void janas_expert_cache_reset_stats(struct janas_expert_cache *c);

#endif