Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
cuda: add lossless compressed expert tier
ZacharyZcR8cc534cbfb10
diff --git a/c/Makefile b/c/Makefileindex 10b1fa46..64bc2790 100644--- a/c/Makefile+++ b/c/Makefile@@ -130,6 +130,8 @@ INSTALL ?= install# clean. See backend_loader.c and README "cuda-dll" below.CUDA ?= 0CUDA_DLL ?= 0+COLI_ANS ?= 0+DIETGPU_ROOT ?=ifneq ($(IS_WIN),)# the CUDA installer sets CUDA_PATH system-wide (e.g. C:\Program Files\NVIDIA# GPU Computing Toolkit\CUDA\v13.2); fall back to it before the POSIX default.@@ -243,6 +245,16 @@ CFLAGS += -DCOLI_CUDALDFLAGS += -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcudart -lstdc++CUDA_OBJ = backend_cuda.oINK_CUDA_OBJ = backend_cuda_ink.o+ifeq ($(COLI_ANS),1)+ifeq ($(strip $(DIETGPU_ROOT)),)+$(error DIETGPU_ROOT is required when COLI_ANS=1)+endif+CFLAGS += -DCOLI_ANS+GPUFLAGS += -DCOLI_ANS -I$(DIETGPU_ROOT) -I$(DIETGPU_ROOT)/build/third_party/glog -I$(DIETGPU_ROOT)/third_party/glog/src+ANS_LIBS = -L$(DIETGPU_ROOT)/build/lib -Wl,-rpath,$(DIETGPU_ROOT)/build/lib -lgpu_ans -ldietgpu_utils -lglog+ANS_NVCC_LIBS = -L$(DIETGPU_ROOT)/build/lib -Xlinker -rpath -Xlinker $(DIETGPU_ROOT)/build/lib -lgpu_ans -ldietgpu_utils -lglog+LDFLAGS += $(ANS_LIBS)+endifendififeq ($(HIP),1)@@ -413,7 +425,7 @@ $(RANSLIB): tools/rans_ctypes.c rans.hcuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)./backend_cuda_test$(EXE)"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)./ragged_attention_test$(EXE)@@ -425,12 +437,12 @@ hip-test:# CI: compile the backend and its test binary WITHOUT executing them (for# runners with the toolchain but no GPU). Pass CUDA_ARCH/HIP=1+HIP_ARCH.gpu-compile: backend_cuda.o- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)cuda-bench: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/bench_tensor_core.cu@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }- "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE) $(ANS_NVCC_LIBS)./backend_cuda_bench$(EXE)olmoe$(EXE): olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.hdiff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex beb8821b..6849a4ea 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -6,6 +6,12 @@#include <cstdlib>#include <cstring>#include <mutex>+#include <vector>++#ifdef COLI_ANS+#include <dietgpu/ans/GpuANSCodec.h>+#include <dietgpu/utils/StackDeviceMemory.h>+#endif#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__)#include <sys/stat.h>@@ -27,10 +33,18 @@ struct ColiCudaTensor {int ng; /* number of scale groups per row = ceil(I/gs) for fmt=4 */size_t scale_count; /* floats in `scales`: O per-row, O*ng grouped */int tracked;+ int weights_owned;+#ifdef COLI_ANS+ size_t archive_bytes;+ int compressed;+#endifRaggedKVEntry ragged[512];int ragged_count;};+#ifdef COLI_ANS+struct AnsArenaChunk { uint8_t *p; size_t used,cap; };+#endiftypedef struct {int device;int compute_major,compute_minor;@@ -46,6 +60,11 @@ typedef struct {void *group_desc; size_t group_desc_cap;size_t tensor_count, tensor_bytes;int group_pending; size_t group_pending_bytes; /* async expert-group in flight (Inc.4) */+#ifdef COLI_ANS+ void *ans_raw; size_t ans_raw_cap;+ dietgpu::StackDeviceMemory *ans_scratch;+ std::vector<AnsArenaChunk> *ans_chunks;+#endif} DeviceContext;typedef struct {@@ -59,6 +78,10 @@ static int g_nctx;static uint64_t g_group_calls,g_group_experts,g_group_rows;static double g_group_h2d_ms,g_group_kernel_ms,g_group_d2h_ms;static std::mutex g_group_stats_mu;+#ifdef COLI_ANS+static FILE *g_ans_sidecar;+static int g_ans_sidecar_pack;+#endifstatic int cuda_ok(cudaError_t err, const char *what) {if (err == cudaSuccess) return 1;@@ -608,6 +631,50 @@ static int reserve_pinned(float **ptr,size_t *cap,size_t bytes){if(!cuda_ok(cudaMallocHost(ptr,bytes),"pinned staging allocation"))return 0;*cap=bytes;return 1;}+#ifdef COLI_ANS+static void *ans_arena_alloc(DeviceContext *ctx,size_t bytes){+ bytes=(bytes+255)&~size_t(255);+ if(!ctx->ans_chunks)ctx->ans_chunks=new std::vector<AnsArenaChunk>;+ if(ctx->ans_chunks->empty()||ctx->ans_chunks->back().cap-ctx->ans_chunks->back().used<bytes){+ size_t cap=256ull<<20;if(cap<bytes)cap=bytes;+ uint8_t *p=nullptr;if(!cuda_ok(cudaMalloc(&p,cap),"ANS arena chunk"))return nullptr;+ ctx->ans_chunks->push_back({p,0,cap});+ }+ AnsArenaChunk &c=ctx->ans_chunks->back();void *p=c.p+c.used;c.used+=bytes;return p;+}+static int prepare_group_weights(DeviceContext *ctx,+ ColiCudaTensor *const *gates,ColiCudaTensor *const *ups,+ ColiCudaTensor *const *downs,int count,GroupDesc *host){+ int n=0; size_t total=0;+ for(int c=0;c<count;c++){+ ColiCudaTensor *q[3]={gates[c],ups[c],downs[c]};+ for(int k=0;k<3;k++) if(q[k]->compressed){n++;total+=q[k]->weight_bytes;}+ }+ if(!n) return 1;+ if(!ctx->ans_scratch||!reserve_bytes(&ctx->ans_raw,&ctx->ans_raw_cap,total)) return 0;+ std::vector<const void*> in; in.reserve(n);+ std::vector<void*> out; out.reserve(n);+ std::vector<uint32_t> cap; cap.reserve(n);+ size_t off=0;+ for(int c=0;c<count;c++){+ ColiCudaTensor *q[3]={gates[c],ups[c],downs[c]};+ const void **dst[3]={&host[c].g,&host[c].u,&host[c].d};+ for(int k=0;k<3;k++) if(q[k]->compressed){+ void *raw=(uint8_t*)ctx->ans_raw+off;+ in.push_back(q[k]->weights);out.push_back(raw);cap.push_back((uint32_t)q[k]->weight_bytes);+ *dst[k]=raw;off+=q[k]->weight_bytes;+ }+ }+ dietgpu::ANSCodecConfig config(11,false);+ dietgpu::ansDecodeBatchPointer(*ctx->ans_scratch,config,(uint32_t)n,in.data(),out.data(),+ cap.data(),nullptr,nullptr,ctx->stream);+ return cuda_ok(cudaGetLastError(),"ANS expert decode launch");+}+#else+static int prepare_group_weights(DeviceContext *,ColiCudaTensor *const *,+ ColiCudaTensor *const *,ColiCudaTensor *const *,int,GroupDesc *){return 1;}+#endif+/* Publish quant.h's E8 codebook to every configured device. __constant__ memory* is per-device, so this walks the contexts; the engine calls it once after init* rather than the backend carrying a second copy of the table that could drift@@ -662,6 +729,12 @@ extern "C" int coli_cuda_init(const int *devices, int count) {if(!cuda_ok(cudaStreamCreateWithFlags(&ctx->stream,cudaStreamNonBlocking),"stream creation")){g_nctx=0;return 0;}+#ifdef COLI_ANS+ if(std::getenv("CUDA_RAW_EXPERTS")){+ ctx->ans_scratch = new dietgpu::StackDeviceMemory(device, 1ull << 30);+ ctx->ans_chunks = new std::vector<AnsArenaChunk>;+ }+#endifg_nctx++;std::fprintf(stderr, "[CUDA] device %d: %s, %.1f GB VRAM, sm_%d%d\n",device, prop.name, prop.totalGlobalMem / 1e9, prop.major, prop.minor);@@ -686,6 +759,12 @@ extern "C" void coli_cuda_shutdown(void) {if (ctx->host_kv) cudaFreeHost(ctx->host_kv);if (ctx->stream) cudaStreamDestroy(ctx->stream);if (ctx->group_desc) cudaFree(ctx->group_desc);+#ifdef COLI_ANS+ if (ctx->ans_raw) cudaFree(ctx->ans_raw);+ if(ctx->ans_chunks){for(auto &c:*ctx->ans_chunks)cudaFree(c.p);delete ctx->ans_chunks;}+ delete ctx->ans_scratch;+ ctx->ans_scratch=nullptr;ctx->ans_chunks=nullptr;ctx->ans_raw=nullptr;ctx->ans_raw_cap=0;+#endifctx->x = ctx->y = ctx->gate = ctx->up = nullptr;ctx->qx=nullptr; ctx->qscale=nullptr;ctx->aq=ctx->al=ctx->ar=ctx->ac=nullptr;@@ -697,6 +776,9 @@ extern "C" void coli_cuda_shutdown(void) {ctx->group_desc=nullptr; ctx->group_desc_cap=0;}g_nctx = 0;+#ifdef COLI_ANS+ if(g_ans_sidecar){std::fclose(g_ans_sidecar);g_ans_sidecar=nullptr;}+#endif}extern "C" int coli_cuda_device_count(void) { return g_nctx; }@@ -776,6 +858,7 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,coli_cuda_tensor_free(t);return 0;}+ t->weights_owned=1;if(fmt==2||fmt==4){ /* same nibble layout: offset-binary -> signed in place */offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes);if(!cuda_ok(cudaGetLastError(),"int4 weight conversion")){coli_cuda_tensor_free(t);return 0;}}@@ -802,10 +885,105 @@ extern "C" int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,return r;}+#ifdef COLI_ANS+struct AnsSidecarHeader {+ uint32_t magic,raw_bytes,archive_bytes,fmt,I,O;+};+static FILE *ans_sidecar(void){+ if(g_ans_sidecar) return g_ans_sidecar;+ const char *path=std::getenv("COLI_ANS_SIDECAR");+ if(!path||!*path) return nullptr;+ g_ans_sidecar_pack=std::getenv("COLI_ANS_PACK")&&std::atoi(std::getenv("COLI_ANS_PACK"));+ g_ans_sidecar=std::fopen(path,g_ans_sidecar_pack?"wb":"rb");+ return g_ans_sidecar;+}+extern "C" int coli_cuda_tensor_upload_compressed(ColiCudaTensor **tensor,+ const void *weights,const float *scales,int fmt,int I,int O,int device){+ if(fmt!=2 || !tensor || *tensor) return 0; /* prototype: per-row int4 experts only */+ FILE *sidecar=ans_sidecar();+ if(sidecar&&!g_ans_sidecar_pack){+ AnsSidecarHeader h{};+ size_t expected=((size_t)I+1)/2*(size_t)O;+ if(std::fread(&h,sizeof(h),1,sidecar)!=1||h.magic!=0x31534e41u||+ h.fmt!=(uint32_t)fmt||h.I!=(uint32_t)I||h.O!=(uint32_t)O||+ expected>UINT32_MAX||h.raw_bytes!=(uint32_t)expected||+ !h.archive_bytes||h.archive_bytes>dietgpu::getMaxCompressedSize(h.raw_bytes)){+ std::fprintf(stderr,"[ANS] invalid or mismatched sidecar record\n");+ return 0;+ }+ std::vector<uint8_t> archive(h.archive_bytes);+ if(std::fread(archive.data(),archive.size(),1,sidecar)!=1){+ std::fprintf(stderr,"[ANS] truncated sidecar record\n");+ return 0;+ }+ DeviceContext *ctx=find_ctx(device); if(!ctx||!select_ctx(ctx)) return 0;+ ColiCudaTensor *t=(ColiCudaTensor*)std::calloc(1,sizeof(*t)); if(!t)return 0;+ t->fmt=fmt;t->I=I;t->O=O;t->device=device;t->weight_bytes=h.raw_bytes;+ t->scale_count=(size_t)O;t->archive_bytes=h.archive_bytes;t->compressed=1;+ t->weights=ans_arena_alloc(ctx,h.archive_bytes);+ if(!t->weights||+ !cuda_ok(cudaMemcpy(t->weights,archive.data(),h.archive_bytes,cudaMemcpyHostToDevice),+ "ANS sidecar upload")||+ !cuda_ok(cudaMalloc(&t->scales,(size_t)O*sizeof(float)),"ANS sidecar scales")||+ !cuda_ok(cudaMemcpy(t->scales,scales,(size_t)O*sizeof(float),cudaMemcpyHostToDevice),+ "ANS sidecar scale upload")){+ coli_cuda_tensor_free(t);return 0;+ }+ t->tracked=1;ctx->tensor_count++;ctx->tensor_bytes+=h.archive_bytes+(size_t)O*sizeof(float);+ *tensor=t;return 1;+ }+ if(!sidecar||!g_ans_sidecar_pack) return 0;+ if(!coli_cuda_tensor_upload(tensor,weights,scales,fmt,I,O,device)) return 0;+ ColiCudaTensor *t=*tensor;+ DeviceContext *ctx=find_ctx(device);+ if(!ctx||!ctx->ans_scratch||!select_ctx(ctx)){ coli_cuda_tensor_free(t);*tensor=nullptr;return 0; }+ uint32_t raw=(uint32_t)t->weight_bytes;+ uint32_t bound=dietgpu::getMaxCompressedSize(raw), *dsize=nullptr;+ void *tmp=nullptr;+ if(!cuda_ok(cudaMalloc(&tmp,bound),"ANS archive allocation")||+ !cuda_ok(cudaMalloc(&dsize,sizeof(*dsize)),"ANS size allocation")){+ if(tmp)cudaFree(tmp);if(dsize)cudaFree(dsize);coli_cuda_tensor_free(t);*tensor=nullptr;return 0;+ }+ dietgpu::ANSCodecConfig config(11,false);+ /* tensor_upload converted offset-binary nibbles on the legacy stream.+ * ctx->stream is explicitly non-blocking, so it does not inherit the+ * legacy-stream dependency. Finish that one-time conversion before the+ * encoder reads the bytes. */+ if(!cuda_ok(cudaStreamSynchronize(0),"ANS source conversion synchronize")){+ cudaFree(tmp);cudaFree(dsize);coli_cuda_tensor_free(t);*tensor=nullptr;return 0;+ }+ dietgpu::ansEncodeBatchStride(*ctx->ans_scratch,config,1,t->weights,raw,raw,nullptr,+ tmp,bound,dsize,ctx->stream);+ uint32_t used=0;+ int ok=cuda_ok(cudaMemcpyAsync(&used,dsize,sizeof(used),cudaMemcpyDeviceToHost,ctx->stream),+ "ANS size download")&&+ cuda_ok(cudaStreamSynchronize(ctx->stream),"ANS encode synchronize")&&used>0&&used<raw;+ cudaFree(dsize);+ if(!ok){cudaFree(tmp);coli_cuda_tensor_free(t);*tensor=nullptr;return 0;}+ if(g_ans_sidecar_pack){+ std::vector<uint8_t> archive(used);+ AnsSidecarHeader h{0x31534e41u,raw,used,(uint32_t)fmt,(uint32_t)I,(uint32_t)O};+ ok=cuda_ok(cudaMemcpy(archive.data(),tmp,used,cudaMemcpyDeviceToHost),"ANS sidecar download")&&+ std::fwrite(&h,sizeof(h),1,sidecar)==1&&+ std::fwrite(archive.data(),archive.size(),1,sidecar)==1;+ cudaFree(tmp);coli_cuda_tensor_free(t);*tensor=nullptr;+ if(!ok)return 0;+ t=(ColiCudaTensor*)std::calloc(1,sizeof(*t));if(!t)return 0;+ t->fmt=fmt;t->I=I;t->O=O;t->device=device;t->weight_bytes=raw;+ t->archive_bytes=used;t->compressed=1;*tensor=t;+ return 1;+ }+ return 0;+}+#endif+extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor,const void *weights,const float *scales) {if (!tensor || !weights || (tensor->fmt && tensor->fmt != 6 && !scales)) return 0;+#ifdef COLI_ANS+ if(tensor->compressed) return 0;+#endifDeviceContext *ctx=find_ctx(tensor->device);if (!select_ctx(ctx)) return 0;if (!cuda_ok(cudaMemcpy(tensor->weights,weights,tensor->weight_bytes,@@ -815,7 +993,6 @@ extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor,(uint8_t*)tensor->weights,tensor->weight_bytes);if(!cuda_ok(cudaGetLastError(),"int4 weight refresh")) return 0;}- int ng = tensor->ng > 0 ? tensor->ng : 1;/* fmt=6 has no scale buffer at all (scales live in-block, scale_count 0), and* the fallback below would otherwise copy O floats out of a NULL host pointer. */return !tensor->fmt || tensor->fmt==6 || cuda_ok(cudaMemcpy(tensor->scales,scales,@@ -954,6 +1131,7 @@ extern "C" int coli_cuda_expert_group(ColiCudaTensor *const *gates,total+=rows[c]; if(rows[c]>max_rows) max_rows=rows[c];}DeviceContext *ctx=find_ctx(device); if(!select_ctx(ctx)) return 0;+ if(!prepare_group_weights(ctx,gates,ups,downs,count,host)) return 0;size_t xb=(size_t)total*D*sizeof(float), ib=(size_t)total*I*sizeof(float);if(!reserve(&ctx->x,&ctx->x_cap,xb)||!reserve(&ctx->y,&ctx->y_cap,xb)||!reserve(&ctx->gate,&ctx->gate_cap,ib)||!reserve(&ctx->up,&ctx->up_cap,ib)||@@ -1114,6 +1292,7 @@ extern "C" int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,}if(total>8) return 0; /* decode-scale only */DeviceContext *ctx=find_ctx(device); if(!ctx||ctx->group_pending||!select_ctx(ctx)) return 0;+ if(!prepare_group_weights(ctx,gates,ups,downs,count,host)) return 0;size_t xb=(size_t)total*D*sizeof(float), ib=(size_t)total*I*sizeof(float);if(!reserve(&ctx->x,&ctx->x_cap,xb)||!reserve(&ctx->y,&ctx->y_cap,xb)||!reserve(&ctx->gate,&ctx->gate_cap,ib)||!reserve(&ctx->up,&ctx->up_cap,ib)||@@ -1330,12 +1509,17 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {/* Must mirror the upload's accounting exactly: fmt=6 never charged for a* scale buffer, and over-subtracting here trips the >= guard below, which* silently leaves the tensor's bytes on the device counter forever. */- size_t bytes = tensor->weight_bytes ++ size_t storage_bytes =+#ifdef COLI_ANS+ tensor->compressed ? tensor->archive_bytes :+#endif+ tensor->weight_bytes;+ size_t bytes = storage_bytes +((tensor->fmt && tensor->fmt != 6) ? (size_t)tensor->O * ng * sizeof(float) : 0);if (ctx->tensor_count) ctx->tensor_count--;if (ctx->tensor_bytes >= bytes) ctx->tensor_bytes -= bytes;}- if (tensor->weights) cudaFree(tensor->weights);+ if (tensor->weights&&tensor->weights_owned) cudaFree(tensor->weights);if (tensor->scales) cudaFree(tensor->scales);for(int i=0;i<tensor->ragged_count;i++){if(tensor->ragged[i].latent)cudaFree(tensor->ragged[i].latent);@@ -1347,7 +1531,12 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {extern "C" size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor) {if (!tensor) return 0;int ng = tensor->ng > 0 ? tensor->ng : 1;- return tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * ng * sizeof(float) : 0);+ size_t storage_bytes =+#ifdef COLI_ANS+ tensor->compressed ? tensor->archive_bytes :+#endif+ tensor->weight_bytes;+ return storage_bytes + (tensor->fmt ? (size_t)tensor->O * ng * sizeof(float) : 0);}extern "C" int coli_cuda_tensor_device(const ColiCudaTensor *tensor) {@@ -1591,6 +1780,7 @@ extern "C" int coli_cuda_expert_group_resident_issue(ColiCudaTensor *const *gate}if(!all_s4) return 0; /* resident path: per-row int4 only */DeviceContext *ctx=find_ctx(device); if(!select_ctx(ctx)) return 0;+ if(!prepare_group_weights(ctx,gates,ups,downs,count,host)) return 0;if(!ctx->ev_done_ok){if(!cuda_ok(cudaEventCreateWithFlags(&ctx->ev_done,cudaEventDisableTiming),"resident group event")) return 0;diff --git a/c/backend_cuda.h b/c/backend_cuda.hindex 7cbb943c..9e2fc86a 100644--- a/c/backend_cuda.h+++ b/c/backend_cuda.h@@ -48,6 +48,14 @@ COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload(ColiCudaTensor **tensor,const void *weights, const float *scales,int fmt, int I, int O, int device);+#ifdef COLI_ANS+/* Experimental Linux-only GPU-resident entropy tier. The archive remains in+ * VRAM and is decoded into per-device scratch immediately before a grouped+ * expert launch. */+COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_compressed(ColiCudaTensor **tensor,+ const void *weights, const float *scales,+ int fmt, int I, int O, int device);+#endif/** y[S,O] = x[S,I] @ W[O,I]^T.diff --git a/c/colibri.c b/c/colibri.cindex 909508c6..eb95b909 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -353,6 +353,7 @@ static int g_cuda_expert_auto;static int g_cuda_dense;static int g_cuda_release_host;static double g_cuda_reserve_gb; /* CUDA_RESERVE_GB: VRAM headroom kept free of expert tier (default 2 GB) */+static int g_cuda_raw_experts=-1; /* experimental ANS tier: keep this global hot prefix raw */static int g_cuda_devices[COLI_CUDA_MAX_DEVICES], g_cuda_ndev, g_cuda_rr;static int64_t g_cuda_dense_projected[COLI_CUDA_MAX_DEVICES];static void qt_cuda_reset(QT *t){@@ -416,6 +417,12 @@ static int qt_cuda_upload(QT *t){return coli_cuda_tensor_upload_g(&t->cuda,weights,t->s,t->fmt,t->I,t->O,t->cuda_device,t->gs);return coli_cuda_tensor_upload(&t->cuda,weights,t->s,t->fmt,t->I,t->O,t->cuda_device);}+#ifdef COLI_ANS+static int qt_cuda_upload_compressed(QT *t){+ if(t->fmt!=2) return 0;+ return coli_cuda_tensor_upload_compressed(&t->cuda,t->q4,t->s,t->fmt,t->I,t->O,t->cuda_device);+}+#endifstatic int qt_cuda_update(QT *t){const void *weights=t->fmt==0?(const void*)t->qf:t->fmt==1?(const void*)t->q8:(const void*)t->q4;@@ -7315,6 +7322,13 @@ static void pin_load(Model *m, const char *statspath, double gb){if(g_cuda_expert_auto) budget=safe_total;if(g_cuda_enabled&&g_cuda_release_host&&budget>0){prefix_est=(int)(budget/eb)+g_cuda_ndev;+#ifdef COLI_ANS+ if(g_cuda_raw_experts>=0){+ int raw=g_cuda_raw_experts;+ if((double)raw*eb>budget) raw=(int)(budget/eb);+ prefix_est=raw+(int)((budget-(double)raw*eb)/(0.80*eb))+g_cuda_ndev;+ }+#endifnpin+=prefix_est; /* additive: prefix RAM is returned after upload */}#endif@@ -7356,17 +7370,28 @@ static void pin_load(Model *m, const char *statspath, double gb){int li=r[a].l;{ ESlot *s=&m->pin[li][slot_of[a]];int64_t need=qt_bytes(&s->g)+qt_bytes(&s->u)+qt_bytes(&s->d);- if(m->gpu_expert_bytes+need>budget) break;+ int compress=0;+#ifdef COLI_ANS+ compress=g_cuda_raw_experts>=0&&a>=g_cuda_raw_experts;+#endif+ int64_t projected=compress?need*80/100:need;+ if(m->gpu_expert_bytes+projected>budget) break;int tried[COLI_CUDA_MAX_DEVICES]={0}, placed=0;for(int attempt=0;attempt<g_cuda_ndev && !placed;attempt++){int best=-1;- for(int i=0;i<g_cuda_ndev;i++) if(!tried[i] && remaining[i]>=need &&+ for(int i=0;i<g_cuda_ndev;i++) if(!tried[i] && remaining[i]>=projected &&(best<0||placed_b[i]<placed_b[best])) best=i;if(best<0) break;tried[best]=1;s->g.cuda_device=s->u.cuda_device=s->d.cuda_device=g_cuda_devices[best];s->g.cuda_eligible=s->u.cuda_eligible=s->d.cuda_eligible=1;- if(qt_cuda_upload(&s->g) && qt_cuda_upload(&s->u) && qt_cuda_upload(&s->d)){+ int uploaded=+#ifdef COLI_ANS+ compress ? (qt_cuda_upload_compressed(&s->g)&&qt_cuda_upload_compressed(&s->u)&&+ qt_cuda_upload_compressed(&s->d)) :+#endif+ (qt_cuda_upload(&s->g) && qt_cuda_upload(&s->u) && qt_cuda_upload(&s->d));+ if(uploaded){int64_t actual=(int64_t)coli_cuda_tensor_bytes(s->g.cuda)+(int64_t)coli_cuda_tensor_bytes(s->u.cuda)+(int64_t)coli_cuda_tensor_bytes(s->d.cuda);@@ -7901,8 +7926,22 @@ int main(int argc, char **argv){g_cuda_expert_auto=cuda_expert&&!strcmp(cuda_expert,"auto");g_cuda_expert_gb=cuda_expert&&!g_cuda_expert_auto?atof(cuda_expert):0;g_cuda_reserve_gb=getenv("CUDA_RESERVE_GB")?atof(getenv("CUDA_RESERVE_GB")):2.0;+#ifdef COLI_ANS+ g_cuda_raw_experts=getenv("CUDA_RAW_EXPERTS")?atoi(getenv("CUDA_RAW_EXPERTS")):-1;+ if(g_cuda_raw_experts>=0&&(!getenv("COLI_ANS_SIDECAR")||!*getenv("COLI_ANS_SIDECAR"))){+ fprintf(stderr,"COLI_ANS_SIDECAR is required when CUDA_RAW_EXPERTS is set\n");+ return 1;+ }+ if(g_cuda_raw_experts>=0&&g_repin){+ fprintf(stderr,"REPIN is incompatible with the fixed-order ANS sidecar\n");+ return 1;+ }+#endifif(!getenv("REPIN")&&g_cuda_expert_auto&&getenv("PIN_GB")&&!strcmp(getenv("PIN_GB"),"all")) g_repin=16;+#ifdef COLI_ANS+ if(g_cuda_raw_experts>=0) g_repin=0;+#endif/* CUDA_RELEASE_HOST default: ndev>1 was chosen when the host copy was the* multi-GPU re-upload path. On a SINGLE GPU asked to fill RAM as well* (PIN_GB=all, or a PIN_GB large enough that the two tiers compete), that@@ -8055,6 +8094,13 @@ int main(int argc, char **argv){pin_load(&m,pin,pin_gb&&!strcmp(pin_gb,"all")?-1.0:pin_gb?atof(pin_gb):10.0); /* PIN_GB=all (#80) */}}+#if defined(COLI_CUDA) && defined(COLI_ANS)+ if(getenv("COLI_ANS_PACK")&&atoi(getenv("COLI_ANS_PACK"))){+ fprintf(stderr,"[ANS] sidecar packing complete; exiting before inference\n");+ coli_cuda_shutdown();+ return 0;+ }+#endifif(getenv("COUPLE")&&*getenv("COUPLE")){ /* coupling-scored cross-layer prefetch (#176) */g_couple_k=getenv("COUPLE_K")?atoi(getenv("COUPLE_K")):8;if(g_couple_k<1)g_couple_k=1; if(g_couple_k>32)g_couple_k=32;