Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
A CUDA fallback is a property of the call, not of the tensor
ZacharyZcR0830495b60d2
diff --git a/c/colibri.c b/c/colibri.cindex 3368782d..2723e6f9 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -199,7 +199,15 @@ typedef struct {#ifdef COLI_VULKANColiVkTensor *vk; int vk_eligible; /* resident on the Vulkan expert tier */#endif- int cuda_eligible, cuda_failed, cuda_device; /* resident tensor, never a reused expert slot */+ int cuda_eligible, cuda_device; /* resident tensor, never a reused expert slot */+ /* #767: the row count of the smallest call that has failed on this tensor, or 0 if+ * none has. A CUDA failure here is almost always a scratch cudaMalloc under memory+ * pressure, and that pressure scales with S -- so it is a property of the CALL, not+ * of the tensor. Recording S instead of a boolean lets a narrower call retry: a+ * prefill chunk that OOMs at S=512 does not condemn decode at S=1. A genuine device+ * fault fails at S=1, records 1, and is never retried -- the old behaviour, reached+ * as a special case of the general rule rather than as a separate one. */+ int cuda_fail_s;} QT;static int64_t qt_bytes(const QT *t){ /* byte residenti del tensore */int64_t n=(int64_t)t->O*t->I;@@ -519,22 +527,27 @@ static int g_cuda_devices[COLI_CUDA_MAX_DEVICES], g_cuda_ndev, g_cuda_rr;static int64_t g_cuda_dense_projected[COLI_CUDA_MAX_DEVICES];static void qt_cuda_reset(QT *t){if(t->cuda){ coli_cuda_tensor_free(t->cuda); t->cuda=NULL; }- t->cuda_failed=0;+ t->cuda_fail_s=0;}-/* #687: a resident tensor that fails to upload is disabled PERMANENTLY and silently- * falls back to CPU. The per-tensor line below is easy to lose in a busy log, and the- * end state -- GPU fully allocated, ~0% useful -- reads as healthy. Count them and say- * it once, loudly, naming the knob that actually fixes it. */+/* #687: a resident tensor that fails to upload falls back to CPU silently. The+ * per-tensor line below is easy to lose in a busy log, and the end state -- GPU fully+ * allocated, ~0% useful -- reads as healthy. Count them and say it once, loudly,+ * naming the knob that actually fixes it.+ *+ * Since #767 the fallback is per-width rather than permanent (see cuda_fail_s), so a+ * tensor counted here may recover on its own at a narrower S. The count is still worth+ * shouting about: it means the tier left too little VRAM for the prompt being run. */static int g_cuda_disabled_n;static void cuda_disabled_note(void){enum { CUDA_DISABLED_LOUD_AT = 8 };if(++g_cuda_disabled_n != CUDA_DISABLED_LOUD_AT) return;fprintf(stderr,- "[CUDA] ***** %d resident tensors have been disabled and moved to CPU *****\n"- "[CUDA] The GPU still holds its expert tier, but the dense/attention path is now\n"- "[CUDA] on CPU: this run will be at or below CPU speed while the card stays\n"- "[CUDA] allocated. The usual cause is the expert tier claiming all VRAM and\n"- "[CUDA] leaving nothing for the lazily-uploaded resident tensors.\n"+ "[CUDA] ***** %d resident tensors have fallen back to CPU *****\n"+ "[CUDA] The GPU still holds its expert tier, but the dense/attention path is\n"+ "[CUDA] running on CPU at this prompt width: expect at or below CPU speed while\n"+ "[CUDA] the card stays allocated. They are retried at narrower S, so decode may\n"+ "[CUDA] recover even when prefill does not. The usual cause is the expert tier\n"+ "[CUDA] claiming all VRAM and leaving nothing for the lazily-uploaded tensors.\n""[CUDA] Fix: set an explicit CUDA_EXPERT_GB below the auto value (leave room\n""[CUDA] per device for dense + attention + workspace). See issue #687.\n",CUDA_DISABLED_LOUD_AT);@@ -600,8 +613,8 @@ static void cuda_stats_print(void){/* #687: say it again at the end -- by now the per-tensor lines are thousands of* log lines back, and this is the number that explains a CPU-speed "GPU" run. */if(g_cuda_disabled_n) fprintf(stderr,- "[CUDA] %d tensors ran on CPU after failed uploads: this run did NOT use the GPU for "- "them. Lower CUDA_EXPERT_GB (#687).\n", g_cuda_disabled_n);+ "[CUDA] %d tensors fell back to CPU after failed uploads at least once. Lower "+ "CUDA_EXPERT_GB to leave room for them (#687, #767).\n", g_cuda_disabled_n);if(g_cuda_ndev>1) for(int i=0;i<g_cuda_ndev;i++){coli_cuda_stats(g_cuda_devices[i],&n,&b);fprintf(stderr,"[CUDA] device %d: %zu tensors, %.2f GB\n",g_cuda_devices[i],n,b/1e9);@@ -879,15 +892,18 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot)* cuda_boot): quant_matmul dispatches it by format, deriving the 128x128* block-scale geometry from the dims alone. Without the LUT (old DLL) it is* excluded up front — same silent-and-immediate idiom as fmt=5. */- if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && w->fmt!=5 &&- (w->fmt!=8 || g_cuda_fp8_ready) && !omp_in_parallel()){+ if(g_cuda_enabled && w->cuda_eligible && (!w->cuda_fail_s || S < w->cuda_fail_s) &&+ w->fmt!=5 && (w->fmt!=8 || g_cuda_fp8_ready) && !omp_in_parallel()){const void *weights = w->fmt==0 ? (const void*)w->qf: (w->fmt==1||w->fmt==8) ? (const void*)w->q8 : (const void*)w->q4;- if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device,w->gs)) return;- w->cuda_failed=1;+ if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device,w->gs)){+ w->cuda_fail_s=0; /* it fits again: forget the width that did not */+ return;+ }+ w->cuda_fail_s = S;if(g_cuda_disabled_n < 8) /* keep the detail for the first few, then the summary */- fprintf(stderr,"[CUDA] tensor [%d,%d] on device %d disabled after an error; falling back to CPU\n",- w->O,w->I,w->cuda_device);+ fprintf(stderr,"[CUDA] tensor [%d,%d] on device %d fell back to CPU at S=%d; "+ "narrower calls will still try the GPU\n", w->O,w->I,w->cuda_device,S);cuda_disabled_note();}#endif