37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* tuner.h - the engine tunes itself on real passes.
*
* Every configuration (how many threads, whether the GPU takes part) gives
* the same results bit for bit, so the engine can try them on the passes it
* runs anyway: for each kind of pass (one token, a few, a block) it tries
* each candidate a few times, keeps the fastest, and now and then tries the
* others again to follow the machine's state (heat, power source). The
* number of threads is settled by the first tries, and moves only when
* other threads win several tries in a row, or one by half or more, each
* a fresh pass against the choice's recent mean: a pass costs more as the
* context grows, so a candidate's old mean, made at short context, must not
* decide - which moved chats onto fewer cores than they could use - but a
* laptop that warms up can turn the first tries' verdict round. The choices are
* saved per machine and model, so the next start begins from them.
*/
#ifndef JANAS_LLM_TUNER_H
#define JANAS_LLM_TUNER_H
#include <stddef.h>
#include <stdint.h>
/* one token; 2-16 (verification); blocks of JANAS_TUNE_BLOCK and more;
17 to JANAS_TUNE_BLOCK - 1 (a chat's short turns). The short blocks
apart since the fast prompt (from JANAS_GPU_FAST_N = 64 tokens) puts
the long ones nearly all on the GPU: together, their mean chose 20
threads for Qwen3.5-2B's prompts, which ran faster on 12 (10 Oct 2026) */
#define JANAS_TUNE_CLASSES 4
#define JANAS_TUNE_BLOCK 64
#define JANAS_TUNE_MAX_CANDS 16
struct janas_tune_cand {
int threads; /* compute threads (a prefix of the pool) */
int gpu; /* the GPU takes part in the products */
int npu; /* the NPU takes the blocks' products it can (blocks only) */
};
/* " + GPU", " + NPU", " + GPU + NPU" or "": a candidate's devices */
const char *janas_tune_devices(const struct janas_tune_cand *c);
struct janas_tuner {
struct janas_tune_cand cand[JANAS_TUNE_MAX_CANDS];
int n_cand;
int fixed; /* no tuning: the default candidate of each class */
int allow_gpu; /* candidates with the GPU may be chosen */
int gpu_one; /* ... for one token too (the GPU takes single tokens) */
int forced; /* >= 0: every pass uses this candidate (and is measured) */
int trying; /* the last pick an alternative tried (exploring, or now
and then), not the class's choice */
/* the combinations of devices a class may use (common/devices.h: bit
gpu | npu << 1), the user's choice; 0 any */
int allowed[JANAS_TUNE_CLASSES];
struct {
double cost[JANAS_TUNE_MAX_CANDS]; /* seconds per token, averaged */
uint32_t tried[JANAS_TUNE_MAX_CANDS];
uint32_t last[JANAS_TUNE_MAX_CANDS]; /* pass of the last try */
/* tries in a row won against the choice by each candidate, and by
any with other threads (their threads taken over when either
reaches RESETTLE_WINS, or at once by a try twice as fast) */
uint8_t wins[JANAS_TUNE_MAX_CANDS], other_wins;
uint32_t passes;
int def; /* the default candidate */
/* the next try of an alternative, the passes until the one after,
and the best candidate when the last try was made (-1: none
pending): a try that changes nothing doubles the gap */
uint32_t retry_at, retry_gap;
int retry_best, retry_cand; /* and the candidate tried */
int threads; /* settled by the first tries (0: not yet) */
} cls[JANAS_TUNE_CLASSES];
};
int janas_tune_class(uint32_t n_tokens);
/* The candidates and each class's default (index into cand). */
void janas_tuner_init(struct janas_tuner *t, const struct janas_tune_cand *c,
int n, const int def[JANAS_TUNE_CLASSES], int fixed);
/* The candidate for the next pass of this class, and its measured cost. */
int janas_tuner_pick(struct janas_tuner *t, int cls);
void janas_tuner_record(struct janas_tuner *t, int cls, int cand,
double sec_per_token);
/* Every pass on candidate cand (measured all the same), -1 to stop. */
void janas_tuner_force(struct janas_tuner *t, int cand);
/* The combinations of devices class cls may use (common/devices.h), 0
any: the user's --prompt-on, --drafts-on, --reply-on. */
void janas_tuner_allow(struct janas_tuner *t, int cls, int mask);
/* Whether candidates using the GPU may be chosen (energy saving: no). */
void janas_tuner_allow_gpu(struct janas_tuner *t, int allow);
/* Whether they may be chosen for one token too: only where the GPU takes
single-token products (janas_gpu_min_n 1, a discrete GPU). */
void janas_tuner_allow_gpu_one(struct janas_tuner *t, int allow);
/* The best known candidate of a class (for work that is not measured). */
int janas_tuner_best(const struct janas_tuner *t, int cls);
/*
* ~/.cache/janas (XDG_CACHE_HOME honoured), where the engine keeps what it
* learns about this machine. With create the directory is made if missing.
* Returns 0, or -1 if there is no place to put it.
*/
int janas_cache_dir(char *path, size_t len, int create);
/* Saved choices, under a key naming machine, model and candidates. */
void janas_tuner_load(struct janas_tuner *t, const char *key);
void janas_tuner_save(const struct janas_tuner *t, const char *key);
/* A one-line summary per class, for reports. */
void janas_tuner_report(const struct janas_tuner *t, char *buf, size_t len);
#endif