Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
use async and fix workers
Robert Landerscf062010d8df
diff --git a/.gitignore b/.gitignoreindex dba6a3fe..981e7400 100644--- a/.gitignore+++ b/.gitignore@@ -67,3 +67,4 @@ c/bench/c/tests/test_decode_batchc/tests/test_i4_acc512c/tests/test_idot+c/tests/test_uringdiff --git a/c/glm.c b/c/glm.cindex 0237869e..7d307dc8 100644--- a/c/glm.c+++ b/c/glm.c@@ -5259,6 +5259,7 @@ int main(int argc, char **argv){g_pipe = getenv("PIPE")?atoi(getenv("PIPE")):0; /* default OFF: overlap expert load β matmul (byte-identical; reorders I/O). PIPE=1 opts in */g_pipe_nw = getenv("PIPE_WORKERS")?atoi(getenv("PIPE_WORKERS")):8; /* I/O worker threads */if(g_pipe_nw<1) g_pipe_nw=1;+ g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0;g_uring = getenv("URING")?atoi(getenv("URING")):0;if(g_uring){#ifdef __linux__@@ -5267,13 +5268,17 @@ int main(int argc, char **argv){if(uring_batch_init(&g_ub_pipe) || (g_pilot_real&&uring_batch_init(&g_ub_pilot))){fprintf(stderr,"URING=1: io_uring_setup failed: %s\n",strerror(errno)); return 2;}- fprintf(stderr,"[URING] queued expert I/O active (depth=%d%s)\n",URING_REQ_MAX,- g_pilot_real?", batched PILOT_REAL":"");+ unsigned uw=(unsigned)(g_pipe_nw>64?64:g_pipe_nw);+ if(coli_uring_set_workers(&g_ub_pipe.ring,uw) ||+ (g_pilot_real&&coli_uring_set_workers(&g_ub_pilot.ring,uw)))+ fprintf(stderr,"[URING] warning: cannot set io-wq workers=%u: %s\n",uw,strerror(errno));+ fprintf(stderr,"[URING] queued expert I/O active (depth=%d, workers=%u, %s%s)\n",URING_REQ_MAX,uw,+ g_direct?"O_DIRECT":"buffered",g_pilot_real?", batched PILOT_REAL":"");+ if(!g_direct) fprintf(stderr,"[URING] cold NVMe: DIRECT=1 avoids page-cache copy/readahead bottlenecks\n");#elsefprintf(stderr,"URING=1 is supported only on Linux\n"); return 2;#endif}- g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0;g_idot = getenv("IDOT")?atoi(getenv("IDOT")):1; /* 0 = kernel f32 esatti (A/B) */if(getenv("ROUTE_TRACE")&&*getenv("ROUTE_TRACE")){g_route_fp=fopen(getenv("ROUTE_TRACE"),"w");diff --git a/c/tests/test_uring.c b/c/tests/test_uring.cindex 9712f659..e64b9f2f 100644--- a/c/tests/test_uring.c+++ b/c/tests/test_uring.c@@ -84,6 +84,9 @@ int main(void){}close(fd); return fail("io_uring_setup");}+ if(coli_uring_set_workers(&ring,4)){+ coli_uring_close(&ring); close(fd); return fail("io-wq worker limit");+ }for(int i=0;i<N;i++) if(coli_uring_prep_read(&ring,fd,dst[i],SZ,(off_t)i*SZ,(uint64_t)i+1)){coli_uring_close(&ring); close(fd); return fail("prepare read");}diff --git a/c/uring.h b/c/uring.hindex bc0d8172..6235234d 100644--- a/c/uring.h+++ b/c/uring.h@@ -81,6 +81,11 @@ static inline int coli_uring_init(ColiUring *r,unsigned entries){return 0;}+static inline int coli_uring_set_workers(ColiUring *r,unsigned workers){+ unsigned limits[2]={workers,workers}; /* bounded, unbounded io-wq workers */+ return (int)syscall(SYS_io_uring_register,r->fd,IORING_REGISTER_IOWQ_MAX_WORKERS,limits,2);+}+static inline int coli_uring_prep_read(ColiUring *r,int fd,void *buf,size_t len,int64_t off,uint64_t user_data){if(!len || len>UINT32_MAX){ errno=EINVAL; return -1; }@@ -91,6 +96,12 @@ static inline int coli_uring_prep_read(ColiUring *r,int fd,void *buf,size_t len,struct io_uring_sqe *sqe=&r->sqes[idx];memset(sqe,0,sizeof(*sqe));sqe->opcode=IORING_OP_READ;+ /* Cold regular-file reads are allowed to execute inline during+ * io_uring_enter() unless forced async. That serializes the submitter on+ * filesystems without native nonblocking buffered reads and destroys the+ * intended I/O/compute overlap. io-wq gives the ring a real bounded worker+ * pool while CQEs retain completion ordering/ownership here. */+ sqe->flags=IOSQE_ASYNC;sqe->fd=fd;sqe->off=(uint64_t)off;sqe->addr=(uint64_t)(uintptr_t)buf;diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex d280a92f..dab656bb 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -39,8 +39,8 @@ Format: `VAR` β default β effect.| `COLI_METAL_GEMM_MIN` | `16` | Minimum matmul rows to dispatch a GEMM to the GPU (below this, stays on CPU). || `COLI_METAL_SPIN` | off | Keep a GPU keep-alive spinner running (reduces dispatch latency; costs power). || `PIPE` | `0` (off) | Overlap expert disk-load with matmul via I/O worker threads. Byte-identical output; reorders I/O. `PIPE=1` opts in. |-| `PIPE_WORKERS` | `8` | Number of I/O worker threads when `PIPE=1`. Tune to your SSD (fewer avoids over-subscribing cores). |-| `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, replaces blocking loader pthreads and spin waits with batched io_uring SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. |+| `PIPE_WORKERS` | `8` | Number of pthread loaders when `PIPE=1`, or the io-wq worker maximum per ring when `URING=1` (capped at 64). Tune to SSD queue depth and available cores. |+| `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. || `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. Helps sustained NVMe; keeps the zero-copy GPU path. || `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. || `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. |