37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* pool.c - persistent thread pool (see pool.h).
*/
#define _GNU_SOURCE
#include "pool.h"
#include "aligned.h"
#include <pthread.h>
#include <sched.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#if defined(__x86_64__)
#include <immintrin.h>
#define cpu_relax() _mm_pause()
#else
#define cpu_relax() ((void)0)
#endif
#define SPIN_ITERATIONS 200000
/*
* The end of a run is where twenty threads meet, and a single counter that
* every one of them decrements has to travel from core to core once per
* thread, in series. So each thread gets a cache line of its own and writes
* in it the number of the run it has just finished; only the caller reads
* them, and the writes never meet. An empty run at twenty threads went from
* 2.75 us to 0.53 us (22 Sep 2026, job/tests/c/pool_cost.c).
*
* On the engine this is worth almost nothing - the transfers were hiding
* under the caller's own share of the work - but it is what makes it certain
* that dispatching is not where the time goes.
*/
#define LINE 64
struct done_flag {
_Alignas(LINE) atomic_ullong run;
double work; /* seconds inside fn, this thread; the trace only */
char pad[LINE - sizeof(atomic_ullong) - sizeof(double)];
};
/*
* JANAS_POOLTRACE=1: for every call site, how long its runs last, how many
* threads they use and how much of that time is spent waiting at the end of
* the run for the slowest thread. Reading it: a high idle share means the
* work does not divide by the threads; the time between runs, which is the
* caller working alone, is not here - it is the difference between the wall
* of the whole pass and the sum of this column - but in gap_s, by the sites
* of the runs before and after it. Off it costs one test of a static
* variable per run.
*/
struct site {
janas_pool_fn fn;
const char *name;
double wall, work;
unsigned long long runs, threads;
};
static struct site sites[32];
static int n_sites;
/* the caller alone between two runs of a pool, by the sites of the run
before and the run after: where in a pass the time between runs goes.
Per pool (struct janas_pool trace_gap): the caller of the experts' reads
waits for work between its runs, which is not work; the report gives the
pool that ran most, the one the passes are made of */
struct trace_gaps {
double s[32][32];
unsigned long long n[32][32], runs;
};
static unsigned trace_epoch = 1; /* janas_pool_trace_reset: a new count */
static struct janas_pool *traced[8];
static int n_traced;
static pthread_mutex_t traced_lock = PTHREAD_MUTEX_INITIALIZER;
/* read by every worker, so atomic: they all decide the same thing, but a
plain int written by twenty threads is still a race */
static atomic_int trace = -1;
static int tracing(void)
{
int t = atomic_load_explicit(&trace, memory_order_relaxed);
if (t < 0) {
const char *e = getenv("JANAS_POOLTRACE");
t = e && atoi(e) > 0;
atomic_store_explicit(&trace, t, memory_order_relaxed);
}
return t;
}
static double now(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return ts.tv_sec + ts.tv_nsec * 1e-9;
}
struct janas_pool {
int n_threads;
pthread_t *threads;
int *cpus;
janas_pool_fn fn;
void *arg;
/* every janas_pool_run publishes a new ticket: its number times 256
plus its count of active threads, in one atomic value so that a
worker never pairs one run with another run's count */
atomic_ullong ticket;
unsigned long long runs; /* written by the caller only */
struct done_flag *done; /* done[i].run: the last run thread i finished */
atomic_int sleepers; /* threads waiting on a condition variable */
double work_prev; /* the trace: sum of done[].work, last run */
double trace_end; /* ... when the last run ended, its site */
int trace_site;
unsigned trace_epoch;
struct trace_gaps *trace_gap; /* allocated by the first traced run */
atomic_bool stop;
atomic_int spin; /* pause iterations before a worker sleeps */
int active; /* threads that take part in the next runs */
pthread_mutex_t lock;
/*
* Two of them, because a run must not wake a thread it has no work for.
* Threads left out by janas_pool_set_active sleep on wake_idle, which is
* broadcast only when the count grows again; the others wait on wake,
* where a broadcast costs nothing while they are spinning. With one
* condition variable a pool of twenty running twelve woke the other
* eight at every call, and they queued on this mutex to find out they
* were not wanted: 6.72 us a run instead of 1.82, measured 21 Sep 2026.
*/
pthread_cond_t wake, wake_idle;
int last_active; /* to notice that the count has grown */
};
struct worker_arg {
struct janas_pool *p;
int tid;
};
static void *worker(void *v)
{
struct worker_arg wa = *(struct worker_arg *)v;
struct janas_pool *p = wa.p;
free(v);
if (p->cpus)
janas_pin_current_thread(p->cpus[wa.tid]);
unsigned long long seen = 0;
for (;;) {
unsigned long long t;
int spins = 0;
/* a thread left out of the last run sleeps at once, and apart, on
wake_idle: spinning would take resources from the busy thread
sharing its core. Before the first run there is no such thing as
left out - seen is zero - and a thread that took it for one would
sleep where nothing wakes it. */
int idle = seen != 0 && wa.tid >= (int)(seen & 255);
while ((t = atomic_load_explicit(&p->ticket, memory_order_acquire)) ==
seen &&
!atomic_load(&p->stop)) {
if (!idle && ++spins < atomic_load_explicit(&p->spin,
memory_order_relaxed)) {
cpu_relax();
continue;
}
pthread_mutex_lock(&p->lock);
/* published before reading the ticket again: a run that saw no
sleeper had already stored its own */
atomic_fetch_add(&p->sleepers, 1);
while (atomic_load(&p->ticket) == seen && !atomic_load(&p->stop))
pthread_cond_wait(idle ? &p->wake_idle : &p->wake, &p->lock);
atomic_fetch_sub(&p->sleepers, 1);
pthread_mutex_unlock(&p->lock);
}
if (atomic_load(&p->stop))
return NULL;
seen = t;
int active = (int)(t & 255);
if (wa.tid < active) {
double w0 = tracing() ? now() : 0;
p->fn(p->arg, wa.tid, active);
if (tracing())
p->done[wa.tid].work += now() - w0;
atomic_store_explicit(&p->done[wa.tid].run, t >> 8,
memory_order_release);
}
}
}
struct janas_pool *janas_pool_create(int n_threads, const int *cpus)
{
if (n_threads < 1 || n_threads > 255)
return NULL;
struct janas_pool *p = calloc(1, sizeof(*p));
if (!p)
return NULL;
p->n_threads = n_threads;
p->active = n_threads;
atomic_init(&p->ticket, 0);
atomic_init(&p->spin, SPIN_ITERATIONS);
p->threads = calloc((size_t)n_threads, sizeof(pthread_t));
p->done = janas_aligned_alloc(LINE, (size_t)n_threads * sizeof(*p->done));
if (p->done)
memset(p->done, 0, (size_t)n_threads * sizeof(*p->done));
if (cpus) {
p->cpus = malloc((size_t)n_threads * sizeof(int));
if (p->cpus)
for (int i = 0; i < n_threads; i++)
p->cpus[i] = cpus[i];
}
if (!p->threads || !p->done || (cpus && !p->cpus)) {
free(p->threads);
janas_aligned_free(p->done);
free(p->cpus);
free(p);
return NULL;
}
p->last_active = 0;
pthread_mutex_init(&p->lock, NULL);
pthread_cond_init(&p->wake, NULL);
pthread_cond_init(&p->wake_idle, NULL);
for (int i = 1; i < n_threads; i++) {
struct worker_arg *wa = malloc(sizeof(*wa));
if (wa) {
wa->p = p;
wa->tid = i;
}
if (!wa || pthread_create(&p->threads[i], NULL, worker, wa) != 0) {
/* a library does not end its host: the threads started stop,
and the caller hears NULL */
free(wa);
p->n_threads = i;
janas_pool_destroy(p);
return NULL;
}
}
return p;
}
static void pool_trace_run(struct janas_pool *p, janas_pool_fn fn,
const char *name, double t0, double t1, int active);
void janas_pool_run_named(struct janas_pool *p, janas_pool_fn fn, void *arg,
const char *name)
{
p->fn = fn;
p->arg = arg;
int active = p->active;
double t0 = tracing() ? now() : 0;
/*
* The lock and the broadcast are for the threads that are asleep, and
* between two runs a few microseconds apart there are none: they are all
* spinning on the ticket. Publishing the ticket first and then reading
* sleepers, both sequentially consistent so that the two cannot be seen
* out of order, makes the usual run a single store. A thread that decides
* to sleep after that read reads the ticket again under the lock, and
* finds it already changed.
*/
int grew = active > p->last_active; /* threads wanted back: they sleep
apart, on wake_idle */
p->last_active = active;
atomic_store_explicit(&p->ticket, ++p->runs << 8 | (unsigned)active,
memory_order_seq_cst);
if (grew || atomic_load_explicit(&p->sleepers, memory_order_seq_cst)) {
pthread_mutex_lock(&p->lock);
pthread_cond_broadcast(&p->wake);
if (grew)
pthread_cond_broadcast(&p->wake_idle);
pthread_mutex_unlock(&p->lock);
}
double w0 = tracing() ? now() : 0;
fn(arg, 0, active);
if (tracing())
p->done[0].work += now() - w0;
for (int i = 1; i < active; i++)
while (atomic_load_explicit(&p->done[i].run, memory_order_acquire) !=
p->runs)
cpu_relax();
if (tracing())
pool_trace_run(p, fn, name, t0, now(), active);
}
/* The trace of one run, from the caller's thread: nothing here is shared. */
static void pool_trace_run(struct janas_pool *p, janas_pool_fn fn,
const char *name, double t0, double t1, int active)
{
double wall = t1 - t0;
double w = 0;
for (int i = 0; i < active; i++)
w += p->done[i].work;
int i = 0;
while (i < n_sites && sites[i].fn != fn)
i++;
if (i == n_sites) {
if (n_sites == 32)
return;
sites[n_sites].fn = fn;
sites[n_sites].name = name;
n_sites++;
}
sites[i].wall += wall;
sites[i].work += w - p->work_prev;
sites[i].threads += (unsigned long long)active;
sites[i].runs++;
p->work_prev = w;
if (!p->trace_gap) {
pthread_mutex_lock(&traced_lock);
if (n_traced < 8 && (p->trace_gap = calloc(1, sizeof *p->trace_gap)))
traced[n_traced++] = p;
pthread_mutex_unlock(&traced_lock);
if (!p->trace_gap)
return;
}
struct trace_gaps *g = p->trace_gap;
if (p->trace_epoch != trace_epoch) /* a new count: from this run on */
memset(g, 0, sizeof *g);
else {
g->s[p->trace_site][i] += t0 - p->trace_end;
g->n[p->trace_site][i]++;
}
g->runs++;
p->trace_epoch = trace_epoch;
p->trace_end = t1;
p->trace_site = i;
}
void janas_pool_trace_reset(void)
{
for (int i = 0; i < n_sites; i++) {
sites[i].wall = sites[i].work = 0;
sites[i].runs = sites[i].threads = 0;
}
trace_epoch++;
}
void janas_pool_trace_report(const char *what, double wall)
{
if (!tracing())
return;
double tw = 0, tk = 0;
for (int i = 0; i < n_sites; i++) {
tw += sites[i].wall;
tk += sites[i].work;
}
double off = 0; /* thread time offered: wall times the threads used */
for (int i = 0; i < n_sites; i++)
if (sites[i].runs)
off += sites[i].wall * (double)sites[i].threads /
(double)sites[i].runs;
fprintf(stderr, "pool, %s: %.0f ms in runs of %.0f ms", what, tw * 1e3,
wall * 1e3);
if (wall > 0)
fprintf(stderr, " (the caller alone %.1f%%)", 100 * (1 - tw / wall));
fprintf(stderr, "\n%-18s %9s %6s %9s %8s\n", " site", "runs", "threads",
"wall ms", "idle");
for (int i = 0; i < n_sites; i++) {
double o = sites[i].runs ? sites[i].wall * (double)sites[i].threads /
(double)sites[i].runs
: 0.0;
fprintf(stderr, " %-16s %9llu %6.1f %9.1f %7.1f%%\n", sites[i].name,
sites[i].runs,
sites[i].runs ? (double)sites[i].threads / (double)sites[i].runs
: 0,
sites[i].wall * 1e3, o > 0 ? 100 * (o - sites[i].work) / o : 0);
}
fprintf(stderr, " %-16s %9s %6s %9.1f %7.1f%%\n", "all", "", "", tw * 1e3,
off > 0 ? 100 * (off - tk) / off : 0);
/* the caller alone, by the runs it falls between, in the pool that ran
most since the reset: the largest */
struct trace_gaps *g = NULL;
pthread_mutex_lock(&traced_lock);
for (int k = 0; k < n_traced; k++)
if (traced[k]->trace_epoch == trace_epoch &&
(!g || traced[k]->trace_gap->runs > g->runs))
g = traced[k]->trace_gap;
pthread_mutex_unlock(&traced_lock);
if (!g)
return;
double gt = 0;
for (int a = 0; a < n_sites; a++)
for (int b = 0; b < n_sites; b++)
gt += g->s[a][b];
if (gt <= 0)
return;
fprintf(stderr,
"the caller alone between runs, %.0f ms:\n%-36s %9s %9s "
"%9s\n",
gt * 1e3, " after -> before", "times", "ms", "us each");
static char listed[32][32];
memset(listed, 0, sizeof listed);
for (int k = 0; k < 16; k++) {
int ba = -1, bb = -1;
for (int a = 0; a < n_sites; a++)
for (int b = 0; b < n_sites; b++)
if (g->n[a][b] && !listed[a][b] &&
(ba < 0 || g->s[a][b] > g->s[ba][bb]))
ba = a, bb = b;
if (ba < 0 || g->s[ba][bb] < 0.005 * gt)
break;
listed[ba][bb] = 1;
char pair[64];
snprintf(pair, sizeof pair, "%s -> %s", sites[ba].name, sites[bb].name);
fprintf(stderr, " %-34s %9llu %9.1f %9.1f\n", pair, g->n[ba][bb],
g->s[ba][bb] * 1e3, g->s[ba][bb] * 1e6 / g->n[ba][bb]);
}
}
int janas_pool_get_spin(const struct janas_pool *p)
{
return atomic_load_explicit(&((struct janas_pool *)p)->spin,
memory_order_relaxed);
}
void janas_pool_set_spin(struct janas_pool *p, int iterations)
{
atomic_store_explicit(&p->spin, iterations, memory_order_relaxed);
}
int janas_pool_size(const struct janas_pool *p)
{
return p->n_threads;
}
void janas_pool_set_active(struct janas_pool *p, int n)
{
p->active = n < 1 ? 1 : (n > p->n_threads ? p->n_threads : n);
}
int janas_pool_active(const struct janas_pool *p)
{
return p->active;
}
void janas_pool_destroy(struct janas_pool *p)
{
if (!p)
return;
if (p->trace_gap) {
pthread_mutex_lock(&traced_lock);
for (int k = 0; k < n_traced; k++)
if (traced[k] == p)
traced[k] = traced[--n_traced];
pthread_mutex_unlock(&traced_lock);
free(p->trace_gap);
}
pthread_mutex_lock(&p->lock);
atomic_store(&p->stop, true);
pthread_cond_broadcast(&p->wake);
pthread_cond_broadcast(&p->wake_idle);
pthread_mutex_unlock(&p->lock);
for (int i = 1; i < p->n_threads; i++)
pthread_join(p->threads[i], NULL);
pthread_mutex_destroy(&p->lock);
pthread_cond_destroy(&p->wake);
pthread_cond_destroy(&p->wake_idle);
free(p->threads);
janas_aligned_free(p->done);
free(p->cpus);
free(p);
}