Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

use async and fix workers

Robert Landerscf062010d8df
diff --git a/.gitignore b/.gitignore
index dba6a3fe..981e7400 100644
--- a/.gitignore
+++ b/.gitignore
@@ -67,3 +67,4 @@ c/bench/
c/tests/test_decode_batch
c/tests/test_i4_acc512
c/tests/test_idot
+c/tests/test_uring
diff --git a/c/glm.c b/c/glm.c
index 0237869e..7d307dc8 100644
--- a/c/glm.c
+++ b/c/glm.c
@@ -5259,6 +5259,7 @@ int main(int argc, char **argv){
g_pipe = getenv("PIPE")?atoi(getenv("PIPE")):0; /* default OFF: overlap expert load β€– matmul (byte-identical; reorders I/O). PIPE=1 opts in */
g_pipe_nw = getenv("PIPE_WORKERS")?atoi(getenv("PIPE_WORKERS")):8; /* I/O worker threads */
if(g_pipe_nw<1) g_pipe_nw=1;
+ g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0;
g_uring = getenv("URING")?atoi(getenv("URING")):0;
if(g_uring){
#ifdef __linux__
@@ -5267,13 +5268,17 @@ int main(int argc, char **argv){
if(uring_batch_init(&g_ub_pipe) || (g_pilot_real&&uring_batch_init(&g_ub_pilot))){
fprintf(stderr,"URING=1: io_uring_setup failed: %s\n",strerror(errno)); return 2;
}
- fprintf(stderr,"[URING] queued expert I/O active (depth=%d%s)\n",URING_REQ_MAX,
- g_pilot_real?", batched PILOT_REAL":"");
+ unsigned uw=(unsigned)(g_pipe_nw>64?64:g_pipe_nw);
+ if(coli_uring_set_workers(&g_ub_pipe.ring,uw) ||
+ (g_pilot_real&&coli_uring_set_workers(&g_ub_pilot.ring,uw)))
+ fprintf(stderr,"[URING] warning: cannot set io-wq workers=%u: %s\n",uw,strerror(errno));
+ fprintf(stderr,"[URING] queued expert I/O active (depth=%d, workers=%u, %s%s)\n",URING_REQ_MAX,uw,
+ g_direct?"O_DIRECT":"buffered",g_pilot_real?", batched PILOT_REAL":"");
+ if(!g_direct) fprintf(stderr,"[URING] cold NVMe: DIRECT=1 avoids page-cache copy/readahead bottlenecks\n");
#else
fprintf(stderr,"URING=1 is supported only on Linux\n"); return 2;
#endif
}
- g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0;
g_idot = getenv("IDOT")?atoi(getenv("IDOT")):1; /* 0 = kernel f32 esatti (A/B) */
if(getenv("ROUTE_TRACE")&&*getenv("ROUTE_TRACE")){
g_route_fp=fopen(getenv("ROUTE_TRACE"),"w");
diff --git a/c/tests/test_uring.c b/c/tests/test_uring.c
index 9712f659..e64b9f2f 100644
--- a/c/tests/test_uring.c
+++ b/c/tests/test_uring.c
@@ -84,6 +84,9 @@ int main(void){
}
close(fd); return fail("io_uring_setup");
}
+ if(coli_uring_set_workers(&ring,4)){
+ coli_uring_close(&ring); close(fd); return fail("io-wq worker limit");
+ }
for(int i=0;i<N;i++) if(coli_uring_prep_read(&ring,fd,dst[i],SZ,(off_t)i*SZ,(uint64_t)i+1)){
coli_uring_close(&ring); close(fd); return fail("prepare read");
}
diff --git a/c/uring.h b/c/uring.h
index bc0d8172..6235234d 100644
--- a/c/uring.h
+++ b/c/uring.h
@@ -81,6 +81,11 @@ static inline int coli_uring_init(ColiUring *r,unsigned entries){
return 0;
}
+static inline int coli_uring_set_workers(ColiUring *r,unsigned workers){
+ unsigned limits[2]={workers,workers}; /* bounded, unbounded io-wq workers */
+ return (int)syscall(SYS_io_uring_register,r->fd,IORING_REGISTER_IOWQ_MAX_WORKERS,limits,2);
+}
+
static inline int coli_uring_prep_read(ColiUring *r,int fd,void *buf,size_t len,
int64_t off,uint64_t user_data){
if(!len || len>UINT32_MAX){ errno=EINVAL; return -1; }
@@ -91,6 +96,12 @@ static inline int coli_uring_prep_read(ColiUring *r,int fd,void *buf,size_t len,
struct io_uring_sqe *sqe=&r->sqes[idx];
memset(sqe,0,sizeof(*sqe));
sqe->opcode=IORING_OP_READ;
+ /* Cold regular-file reads are allowed to execute inline during
+ * io_uring_enter() unless forced async. That serializes the submitter on
+ * filesystems without native nonblocking buffered reads and destroys the
+ * intended I/O/compute overlap. io-wq gives the ring a real bounded worker
+ * pool while CQEs retain completion ordering/ownership here. */
+ sqe->flags=IOSQE_ASYNC;
sqe->fd=fd;
sqe->off=(uint64_t)off;
sqe->addr=(uint64_t)(uintptr_t)buf;
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index d280a92f..dab656bb 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -39,8 +39,8 @@ Format: `VAR` β€” default β€” effect.
| `COLI_METAL_GEMM_MIN` | `16` | Minimum matmul rows to dispatch a GEMM to the GPU (below this, stays on CPU). |
| `COLI_METAL_SPIN` | off | Keep a GPU keep-alive spinner running (reduces dispatch latency; costs power). |
| `PIPE` | `0` (off) | Overlap expert disk-load with matmul via I/O worker threads. Byte-identical output; reorders I/O. `PIPE=1` opts in. |
-| `PIPE_WORKERS` | `8` | Number of I/O worker threads when `PIPE=1`. Tune to your SSD (fewer avoids over-subscribing cores). |
-| `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, replaces blocking loader pthreads and spin waits with batched io_uring SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. |
+| `PIPE_WORKERS` | `8` | Number of pthread loaders when `PIPE=1`, or the io-wq worker maximum per ring when `URING=1` (capped at 64). Tune to SSD queue depth and available cores. |
+| `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. |
| `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. Helps sustained NVMe; keeps the zero-copy GPU path. |
| `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. |
| `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. |