Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
cuda: pipeline compressed sidecar loading
ZacharyZcR2e4cd63534d2
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 6849a4ea..d674b1fa 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -5,9 +5,16 @@#include <cstdio>#include <cstdlib>#include <cstring>+#include <cerrno>+#include <chrono>#include <mutex>#include <vector>+#if defined(__linux__)+#include <fcntl.h>+#include <unistd.h>+#endif+#ifdef COLI_ANS#include <dietgpu/ans/GpuANSCodec.h>#include <dietgpu/utils/StackDeviceMemory.h>@@ -62,6 +69,8 @@ typedef struct {int group_pending; size_t group_pending_bytes; /* async expert-group in flight (Inc.4) */#ifdef COLI_ANSvoid *ans_raw; size_t ans_raw_cap;+ void *ans_host; size_t ans_host_cap;+ int ans_copy_pending;dietgpu::StackDeviceMemory *ans_scratch;std::vector<AnsArenaChunk> *ans_chunks;#endif@@ -81,6 +90,17 @@ static std::mutex g_group_stats_mu;#ifdef COLI_ANSstatic FILE *g_ans_sidecar;static int g_ans_sidecar_pack;+#if defined(__linux__)+static int g_ans_direct_fd=-1;+static off_t g_ans_direct_off;+#endif+static uint64_t g_ans_load_records;+static double g_ans_header_s,g_ans_read_s,g_ans_stage_s,g_ans_enqueue_s;+static int g_ans_profile_printed;+static double ans_now_s(){+ using clock=std::chrono::steady_clock;+ return std::chrono::duration<double>(clock::now().time_since_epoch()).count();+}#endifstatic int cuda_ok(cudaError_t err, const char *what) {@@ -642,6 +662,17 @@ static void *ans_arena_alloc(DeviceContext *ctx,size_t bytes){}AnsArenaChunk &c=ctx->ans_chunks->back();void *p=c.p+c.used;c.used+=bytes;return p;}+static int ans_host_reserve(DeviceContext *ctx,size_t bytes){+ if(ctx->ans_copy_pending){+ if(!cuda_ok(cudaStreamSynchronize(ctx->stream),"ANS sidecar upload synchronize"))return 0;+ ctx->ans_copy_pending=0;+ }+ if(ctx->ans_host_cap>=bytes)return 1;+ if(ctx->ans_host)cudaFreeHost(ctx->ans_host);+ ctx->ans_host=nullptr;ctx->ans_host_cap=0;+ if(!cuda_ok(cudaMallocHost(&ctx->ans_host,bytes),"ANS pinned staging allocation"))return 0;+ ctx->ans_host_cap=bytes;return 1;+}static int prepare_group_weights(DeviceContext *ctx,ColiCudaTensor *const *gates,ColiCudaTensor *const *ups,ColiCudaTensor *const *downs,int count,GroupDesc *host){@@ -651,6 +682,14 @@ static int prepare_group_weights(DeviceContext *ctx,for(int k=0;k<3;k++) if(q[k]->compressed){n++;total+=q[k]->weight_bytes;}}if(!n) return 1;+ if(!g_ans_profile_printed&&std::getenv("COLI_ANS_PROFILE")){+ g_ans_profile_printed=1;+ std::fprintf(stderr,+ "[ANS] load profile: %llu records | header %.2fs | read %.2fs | "+ "staging/alloc %.2fs | enqueue %.2fs\n",+ (unsigned long long)g_ans_load_records,g_ans_header_s,g_ans_read_s,+ g_ans_stage_s,g_ans_enqueue_s);+ }if(!ctx->ans_scratch||!reserve_bytes(&ctx->ans_raw,&ctx->ans_raw_cap,total)) return 0;std::vector<const void*> in; in.reserve(n);std::vector<void*> out; out.reserve(n);@@ -760,10 +799,13 @@ extern "C" void coli_cuda_shutdown(void) {if (ctx->stream) cudaStreamDestroy(ctx->stream);if (ctx->group_desc) cudaFree(ctx->group_desc);#ifdef COLI_ANS+ if(ctx->ans_copy_pending)cudaStreamSynchronize(ctx->stream);+ if(ctx->ans_host)cudaFreeHost(ctx->ans_host);if (ctx->ans_raw) cudaFree(ctx->ans_raw);if(ctx->ans_chunks){for(auto &c:*ctx->ans_chunks)cudaFree(c.p);delete ctx->ans_chunks;}delete ctx->ans_scratch;ctx->ans_scratch=nullptr;ctx->ans_chunks=nullptr;ctx->ans_raw=nullptr;ctx->ans_raw_cap=0;+ ctx->ans_host=nullptr;ctx->ans_host_cap=0;ctx->ans_copy_pending=0;#endifctx->x = ctx->y = ctx->gate = ctx->up = nullptr;ctx->qx=nullptr; ctx->qscale=nullptr;@@ -778,6 +820,9 @@ extern "C" void coli_cuda_shutdown(void) {g_nctx = 0;#ifdef COLI_ANSif(g_ans_sidecar){std::fclose(g_ans_sidecar);g_ans_sidecar=nullptr;}+#if defined(__linux__)+ if(g_ans_direct_fd>=0){close(g_ans_direct_fd);g_ans_direct_fd=-1;g_ans_direct_off=0;}+#endif#endif}@@ -895,6 +940,15 @@ static FILE *ans_sidecar(void){if(!path||!*path) return nullptr;g_ans_sidecar_pack=std::getenv("COLI_ANS_PACK")&&std::atoi(std::getenv("COLI_ANS_PACK"));g_ans_sidecar=std::fopen(path,g_ans_sidecar_pack?"wb":"rb");+#if defined(__linux__)+ if(g_ans_sidecar&&!g_ans_sidecar_pack&&std::getenv("COLI_ANS_DIRECT")&&+ std::atoi(std::getenv("COLI_ANS_DIRECT"))){+ g_ans_direct_fd=open(path,O_RDONLY|O_DIRECT);+ if(g_ans_direct_fd<0)std::fprintf(stderr,"[ANS] O_DIRECT unavailable; using buffered sidecar\n");+ }+ if(g_ans_sidecar&&!g_ans_sidecar_pack&&g_ans_direct_fd<0)+ posix_fadvise(fileno(g_ans_sidecar),0,0,POSIX_FADV_SEQUENTIAL);+#endifreturn g_ans_sidecar;}extern "C" int coli_cuda_tensor_upload_compressed(ColiCudaTensor **tensor,@@ -904,31 +958,87 @@ extern "C" int coli_cuda_tensor_upload_compressed(ColiCudaTensor **tensor,if(sidecar&&!g_ans_sidecar_pack){AnsSidecarHeader h{};size_t expected=((size_t)I+1)/2*(size_t)O;- if(std::fread(&h,sizeof(h),1,sidecar)!=1||h.magic!=0x31534e41u||+ double t0=ans_now_s();+#if defined(__linux__)+ size_t direct_delta=0,direct_bytes=0,direct_data=0;+ if(g_ans_direct_fd>=0){+ alignas(4096) uint8_t first[8192];+ off_t start=g_ans_direct_off&~off_t(4095);+ direct_delta=(size_t)(g_ans_direct_off-start);+ ssize_t got=pread(g_ans_direct_fd,first,sizeof(first),start);+ if(got<(ssize_t)(direct_delta+sizeof(h))){+ std::fprintf(stderr,"[ANS] direct header read failed at %lld: got %lld errno %d\n",+ (long long)g_ans_direct_off,(long long)got,errno);+ return 0;+ }+ std::memcpy(&h,first+direct_delta,sizeof(h));+ }else+#endif+ if(std::fread(&h,sizeof(h),1,sidecar)!=1)return 0;+ if(h.magic!=0x31534e41u||h.fmt!=(uint32_t)fmt||h.I!=(uint32_t)I||h.O!=(uint32_t)O||expected>UINT32_MAX||h.raw_bytes!=(uint32_t)expected||!h.archive_bytes||h.archive_bytes>dietgpu::getMaxCompressedSize(h.raw_bytes)){std::fprintf(stderr,"[ANS] invalid or mismatched sidecar record\n");return 0;}- std::vector<uint8_t> archive(h.archive_bytes);- if(std::fread(archive.data(),archive.size(),1,sidecar)!=1){- std::fprintf(stderr,"[ANS] truncated sidecar record\n");- return 0;- }+ g_ans_header_s+=ans_now_s()-t0;DeviceContext *ctx=find_ctx(device); if(!ctx||!select_ctx(ctx)) return 0;ColiCudaTensor *t=(ColiCudaTensor*)std::calloc(1,sizeof(*t)); if(!t)return 0;t->fmt=fmt;t->I=I;t->O=O;t->device=device;t->weight_bytes=h.raw_bytes;t->scale_count=(size_t)O;t->archive_bytes=h.archive_bytes;t->compressed=1;+ t0=ans_now_s();t->weights=ans_arena_alloc(ctx,h.archive_bytes);- if(!t->weights||- !cuda_ok(cudaMemcpy(t->weights,archive.data(),h.archive_bytes,cudaMemcpyHostToDevice),- "ANS sidecar upload")||- !cuda_ok(cudaMalloc(&t->scales,(size_t)O*sizeof(float)),"ANS sidecar scales")||- !cuda_ok(cudaMemcpy(t->scales,scales,(size_t)O*sizeof(float),cudaMemcpyHostToDevice),- "ANS sidecar scale upload")){+ size_t scale_bytes=(size_t)O*sizeof(float),scale_off;+#if defined(__linux__)+ if(g_ans_direct_fd>=0){+ size_t record_bytes=sizeof(h)+(size_t)h.archive_bytes;+ direct_data=direct_delta+sizeof(h);+ direct_bytes=(direct_delta+record_bytes+4095)&~size_t(4095);+ scale_off=(direct_bytes+255)&~size_t(255);+ }else+#endif+ scale_off=(h.archive_bytes+255)&~size_t(255);+ if(!t->weights||!ans_host_reserve(ctx,scale_off+scale_bytes)||+ !cuda_ok(cudaMalloc(&t->scales,scale_bytes),"ANS sidecar scales")){+ coli_cuda_tensor_free(t);return 0;+ }+ g_ans_stage_s+=ans_now_s()-t0;+ t0=ans_now_s();+#if defined(__linux__)+ if(g_ans_direct_fd>=0){+ off_t start=g_ans_direct_off&~off_t(4095);+ ssize_t got=pread(g_ans_direct_fd,ctx->ans_host,direct_bytes,start);+ if(got<(ssize_t)(direct_data+h.archive_bytes)){+ std::fprintf(stderr,+ "[ANS] direct record read failed at %lld: need %zu got %lld errno %d\n",+ (long long)g_ans_direct_off,direct_data+h.archive_bytes,+ (long long)got,errno);+ coli_cuda_tensor_free(t);return 0;+ }+ g_ans_direct_off+=(off_t)sizeof(h)+(off_t)h.archive_bytes;+ }else+#endif+ if(std::fread(ctx->ans_host,h.archive_bytes,1,sidecar)!=1){+ std::fprintf(stderr,"[ANS] truncated sidecar record\n");+ coli_cuda_tensor_free(t);return 0;+ }+ g_ans_read_s+=ans_now_s()-t0;+ std::memcpy((uint8_t*)ctx->ans_host+scale_off,scales,scale_bytes);+ t0=ans_now_s();+ void *archive_src=+#if defined(__linux__)+ g_ans_direct_fd>=0?(uint8_t*)ctx->ans_host+direct_data:+#endif+ ctx->ans_host;+ if(!cuda_ok(cudaMemcpyAsync(t->weights,archive_src,h.archive_bytes,+ cudaMemcpyHostToDevice,ctx->stream),"ANS sidecar upload")||+ !cuda_ok(cudaMemcpyAsync(t->scales,(uint8_t*)ctx->ans_host+scale_off,scale_bytes,+ cudaMemcpyHostToDevice,ctx->stream),"ANS sidecar scale upload")){coli_cuda_tensor_free(t);return 0;}+ g_ans_enqueue_s+=ans_now_s()-t0;g_ans_load_records++;+ ctx->ans_copy_pending=1;t->tracked=1;ctx->tensor_count++;ctx->tensor_bytes+=h.archive_bytes+(size_t)O*sizeof(float);*tensor=t;return 1;}diff --git a/c/colibri.c b/c/colibri.cindex eb95b909..e9d1d9e9 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -7391,6 +7391,12 @@ static void pin_load(Model *m, const char *statspath, double gb){qt_cuda_upload_compressed(&s->d)) :#endif(qt_cuda_upload(&s->g) && qt_cuda_upload(&s->u) && qt_cuda_upload(&s->d));+#ifdef COLI_ANS+ if(compress&&!uploaded){+ fprintf(stderr,"[ANS] sidecar load failed; refusing partial VRAM placement\n");+ exit(2);+ }+#endifif(uploaded){int64_t actual=(int64_t)coli_cuda_tensor_bytes(s->g.cuda)+(int64_t)coli_cuda_tensor_bytes(s->u.cuda)diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 47bda964..2bb416e9 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -129,6 +129,8 @@ See [docs/vulkan.md](vulkan.md). On multi-core boxes also set `COLI_NO_OMP_TUNE=| `CUDA_RAW_EXPERTS` | unset | Experimental ANS build only: keep this many hottest experts raw, then store subsequent VRAM experts losslessly compressed. Requires `COLI_ANS_SIDECAR`. || `COLI_ANS_SIDECAR` | unset | Experimental ANS build only: path to the sequential compressed-expert sidecar. || `COLI_ANS_PACK` | `0` | Experimental ANS build only: `=1` creates `COLI_ANS_SIDECAR` during pinning and exits before inference. |+| `COLI_ANS_DIRECT` | `0` | Experimental ANS build on Linux: `=1` reads the sidecar with aligned `O_DIRECT`, bypassing page-cache overhead. Falls back to buffered I/O if unavailable. |+| `COLI_ANS_PROFILE` | `0` | Experimental ANS build: print sidecar header, read, staging/allocation, and H2D enqueue timings on first use. || `COLI_METAL_UNTRACKED` | off (Metal only) | `=1` sets `MTLResourceHazardTrackingModeUntracked` on Metal buffers (reduces hazard-tracking overhead). |> **Windows note.** On Windows, a bare `coli chat` / `coli run` / `coli serve`diff --git a/docs/cuda.md b/docs/cuda.mdindex f99e7f3f..4490a082 100644--- a/docs/cuda.md+++ b/docs/cuda.md@@ -94,15 +94,18 @@ Remove `COLI_ANS_PACK` to load the sidecar. Records are checked against theexpected format, dimensions, raw size, codec bound, and file length. A sidecaris tied to its exact expert placement order; changing the model, profile,budgets, raw-prefix size, or device set requires repacking it.+On Linux, `COLI_ANS_DIRECT=1` uses aligned direct reads while retaining the same+sidecar format. `COLI_ANS_PROFILE=1` prints a one-line load-time breakdown.On a 6Γ RTX 5090 host with GLM-5.2 int4, a 2,500-expert raw tier plus 8,128compressed experts increased VRAM capacity from 9,335 to 10,628 experts(+13.9%). Greedy decode was byte-identical to the raw-tier baseline and measured6.19β7.12 tok/s over 32 tokens (+15.0%), and 6.75β7.31 tok/s over 128 tokens-(+8.3%). The 110 GB sidecar took about 185 seconds to load. These results are a-capacity trade, not a general compression guarantee: incompressible records-are rejected, startup is currently serial, and the archived DietGPU dependency-makes this an experimental build-time option.+(+8.3%). Pinned asynchronous staging reduced placement from 197 to 157 seconds;+aligned direct reads reduced it further to 80β103 seconds across two runs, with+the 110 GB archive read accounting for 18.94β19.15 seconds. These results are a+general compression guarantee: incompressible records are rejected, and the+archived DietGPU dependency makes this an experimental build-time option.Live `REPIN` is disabled because it would invalidate the sidecar's fixed expertorder.