37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* uring.c - asynchronous reads with io_uring (see uring.h).
*
* liburing is not used: the whole of it, for reads, is three system calls and
* two shared rings, and one dependency less is worth the hundred lines. The
* operation is IORING_OP_READV rather than IORING_OP_READ because it is there
* in every kernel that has io_uring at all; with blocks of a megabyte and
* more the one iovec costs nothing measurable.
*/
#define _GNU_SOURCE
#include "common/uring.h"
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#if defined(__linux__)
#include <sys/syscall.h>
#if defined(__NR_io_uring_setup) && defined(__NR_io_uring_enter) && \
defined(__has_include)
#if __has_include(<linux/io_uring.h>)
#define JANAS_HAVE_URING 1
#endif
#endif
#endif
/* Windows: uring_win.c, with overlapped reads */
#if !defined(JANAS_HAVE_URING) && !defined(_WIN32)
struct janas_uring *janas_uring_create(unsigned entries)
{
(void)entries;
return NULL;
}
void janas_uring_destroy(struct janas_uring *u)
{
(void)u;
}
int janas_uring_read(struct janas_uring *u, int fd, void *dst, uint64_t bytes,
uint64_t offset)
{
(void)u, (void)fd, (void)dst, (void)bytes, (void)offset;
return -1;
}
int janas_uring_submit(struct janas_uring *u)
{
(void)u;
return -1;
}
int janas_uring_wait(struct janas_uring *u)
{
(void)u;
return -1;
}
unsigned janas_uring_inflight(const struct janas_uring *u)
{
(void)u;
return 0;
}
#elif defined(JANAS_HAVE_URING)
#include <linux/io_uring.h>
#include <sys/mman.h>
#include <sys/uio.h>
#include <unistd.h>
struct request {
struct iovec iov; /* read by the kernel at submission: it must live here */
int fd;
uint8_t *dst;
uint64_t offset, bytes, done;
};
struct janas_uring {
int fd;
unsigned entries;
void *sq_map, *cq_map, *sqe_map;
size_t sq_size, cq_size, sqe_size;
unsigned *sq_head, *sq_tail, *sq_mask, *sq_array;
unsigned *cq_head, *cq_tail, *cq_mask;
struct io_uring_sqe *sqes;
struct io_uring_cqe *cqes;
struct request *req;
unsigned n_req; /* slots taken by this batch */
unsigned pending; /* of them, not yet complete */
unsigned to_submit; /* queued, not yet handed to the kernel */
};
static int uring_setup(unsigned entries, struct io_uring_params *p)
{
return (int)syscall(__NR_io_uring_setup, entries, p);
}
static int uring_enter(int fd, unsigned to_submit, unsigned min_complete,
unsigned flags)
{
return (int)syscall(__NR_io_uring_enter, fd, to_submit, min_complete, flags,
(void *)NULL, (size_t)0);
}
void janas_uring_destroy(struct janas_uring *u)
{
if (!u)
return;
if (u->sqe_map)
munmap(u->sqe_map, u->sqe_size);
if (u->cq_map && u->cq_map != u->sq_map)
munmap(u->cq_map, u->cq_size);
if (u->sq_map)
munmap(u->sq_map, u->sq_size);
if (u->fd >= 0)
close(u->fd);
free(u->req);
free(u);
}
struct janas_uring *janas_uring_create(unsigned entries)
{
struct io_uring_params p;
memset(&p, 0, sizeof(p));
int fd = uring_setup(entries, &p);
if (fd < 0)
return NULL;
struct janas_uring *u = calloc(1, sizeof(*u));
if (!u) {
close(fd);
return NULL;
}
u->fd = fd;
u->entries = p.sq_entries;
u->sq_size = p.sq_off.array + p.sq_entries * sizeof(unsigned);
u->cq_size = p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe);
if (p.features & IORING_FEAT_SINGLE_MMAP) { /* one mapping for both */
if (u->cq_size > u->sq_size)
u->sq_size = u->cq_size;
u->cq_size = u->sq_size;
}
u->sq_map = mmap(NULL, u->sq_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING);
if (u->sq_map == MAP_FAILED) {
u->sq_map = NULL;
janas_uring_destroy(u);
return NULL;
}
if (p.features & IORING_FEAT_SINGLE_MMAP) {
u->cq_map = u->sq_map;
} else {
u->cq_map = mmap(NULL, u->cq_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_CQ_RING);
if (u->cq_map == MAP_FAILED) {
u->cq_map = NULL;
janas_uring_destroy(u);
return NULL;
}
}
u->sqe_size = p.sq_entries * sizeof(struct io_uring_sqe);
u->sqe_map = mmap(NULL, u->sqe_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES);
if (u->sqe_map == MAP_FAILED) {
u->sqe_map = NULL;
janas_uring_destroy(u);
return NULL;
}
uint8_t *sq = u->sq_map, *cq = u->cq_map;
u->sq_head = (unsigned *)(sq + p.sq_off.head);
u->sq_tail = (unsigned *)(sq + p.sq_off.tail);
u->sq_mask = (unsigned *)(sq + p.sq_off.ring_mask);
u->sq_array = (unsigned *)(sq + p.sq_off.array);
u->cq_head = (unsigned *)(cq + p.cq_off.head);
u->cq_tail = (unsigned *)(cq + p.cq_off.tail);
u->cq_mask = (unsigned *)(cq + p.cq_off.ring_mask);
u->sqes = u->sqe_map;
u->cqes = (struct io_uring_cqe *)(cq + p.cq_off.cqes);
u->req = calloc(u->entries, sizeof(*u->req));
if (!u->req) {
janas_uring_destroy(u);
return NULL;
}
return u;
}
/* Puts the missing part of request i in the submission queue. 0 if full. */
static int queue_sqe(struct janas_uring *u, unsigned i)
{
struct request *r = &u->req[i];
unsigned tail = *u->sq_tail;
unsigned head = __atomic_load_n(u->sq_head, __ATOMIC_ACQUIRE);
if (tail - head >= u->entries)
return 0;
unsigned index = tail & *u->sq_mask;
struct io_uring_sqe *sqe = &u->sqes[index];
memset(sqe, 0, sizeof(*sqe));
r->iov.iov_base = r->dst + r->done;
r->iov.iov_len = (size_t)(r->bytes - r->done);
sqe->opcode = IORING_OP_READV;
/* Without this the kernel runs the read inside io_uring_enter() whenever
it can, which for an O_DIRECT block of a megabyte and more means the
whole read: measured on 20 Sep 2026, 10.5 ms a call for fourteen
experts, 2.3 GB/s, all of it on the thread that was meant to go on
computing. IOSQE_ASYNC hands it to the kernel's own workers instead,
which is the whole point of submitting it. */
sqe->flags = IOSQE_ASYNC;
sqe->fd = r->fd;
sqe->off = r->offset + r->done;
sqe->addr = (uint64_t)(uintptr_t)&r->iov;
sqe->len = 1;
sqe->user_data = i;
u->sq_array[index] = index;
__atomic_store_n(u->sq_tail, tail + 1, __ATOMIC_RELEASE);
u->to_submit++;
return 1;
}
int janas_uring_read(struct janas_uring *u, int fd, void *dst, uint64_t bytes,
uint64_t offset)
{
if (u->n_req >= u->entries)
return -1;
u->req[u->n_req] = (struct request){.iov = {.iov_base = NULL, .iov_len = 0},
.fd = fd,
.dst = dst,
.offset = offset,
.bytes = bytes,
.done = 0};
if (!queue_sqe(u, u->n_req))
return -1;
u->n_req++;
u->pending++;
return 0;
}
int janas_uring_submit(struct janas_uring *u)
{
while (u->to_submit) {
int n = uring_enter(u->fd, u->to_submit, 0, 0);
if (n < 0) {
if (errno == EINTR)
continue;
return -1;
}
if (n == 0) /* the kernel took nothing: leave it to the wait */
break;
u->to_submit -= (unsigned)n;
}
return 0;
}
/* Takes every completion the kernel has published. -1 if a read failed. */
static int reap(struct janas_uring *u)
{
int rc = 0;
unsigned head = *u->cq_head;
unsigned tail = __atomic_load_n(u->cq_tail, __ATOMIC_ACQUIRE);
for (; head != tail; head++) {
struct io_uring_cqe *e = &u->cqes[head & *u->cq_mask];
unsigned i = (unsigned)e->user_data;
int res = e->res;
int again = 0;
if (res > 0) {
u->req[i].done += (uint64_t)res;
again = u->req[i].done < u->req[i].bytes; /* a short read */
} else {
again = res == -EINTR || res == -EAGAIN;
if (!again)
rc = -1;
}
if (again) {
if (!queue_sqe(u, i)) { /* cannot happen: a slot just freed */
rc = -1;
u->pending--;
}
} else {
u->pending--;
}
}
__atomic_store_n(u->cq_head, head, __ATOMIC_RELEASE);
return rc;
}
int janas_uring_wait(struct janas_uring *u)
{
int rc = 0;
/* on a failure the wait goes on all the same: until the kernel has
answered, the buffers of the other reads are still its own */
while (u->pending) {
int n = uring_enter(u->fd, u->to_submit, 1, IORING_ENTER_GETEVENTS);
if (n < 0) {
if (errno == EINTR)
continue;
rc = -1;
break;
}
u->to_submit -= (unsigned)n < u->to_submit ? (unsigned)n : u->to_submit;
if (reap(u) != 0)
rc = -1;
}
u->n_req = 0;
u->pending = 0;
u->to_submit = 0;
return rc;
}
unsigned janas_uring_inflight(const struct janas_uring *u)
{
return u->pending;
}
#endif