Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

A CUDA fallback is a property of the call, not of the tensor

ZacharyZcR0830495b60d2
diff --git a/c/colibri.c b/c/colibri.c
index 3368782d..2723e6f9 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -199,7 +199,15 @@ typedef struct {
#ifdef COLI_VULKAN
ColiVkTensor *vk; int vk_eligible; /* resident on the Vulkan expert tier */
#endif
- int cuda_eligible, cuda_failed, cuda_device; /* resident tensor, never a reused expert slot */
+ int cuda_eligible, cuda_device; /* resident tensor, never a reused expert slot */
+ /* #767: the row count of the smallest call that has failed on this tensor, or 0 if
+ * none has. A CUDA failure here is almost always a scratch cudaMalloc under memory
+ * pressure, and that pressure scales with S -- so it is a property of the CALL, not
+ * of the tensor. Recording S instead of a boolean lets a narrower call retry: a
+ * prefill chunk that OOMs at S=512 does not condemn decode at S=1. A genuine device
+ * fault fails at S=1, records 1, and is never retried -- the old behaviour, reached
+ * as a special case of the general rule rather than as a separate one. */
+ int cuda_fail_s;
} QT;
static int64_t qt_bytes(const QT *t){ /* byte residenti del tensore */
int64_t n=(int64_t)t->O*t->I;
@@ -519,22 +527,27 @@ static int g_cuda_devices[COLI_CUDA_MAX_DEVICES], g_cuda_ndev, g_cuda_rr;
static int64_t g_cuda_dense_projected[COLI_CUDA_MAX_DEVICES];
static void qt_cuda_reset(QT *t){
if(t->cuda){ coli_cuda_tensor_free(t->cuda); t->cuda=NULL; }
- t->cuda_failed=0;
+ t->cuda_fail_s=0;
}
-/* #687: a resident tensor that fails to upload is disabled PERMANENTLY and silently
- * falls back to CPU. The per-tensor line below is easy to lose in a busy log, and the
- * end state -- GPU fully allocated, ~0% useful -- reads as healthy. Count them and say
- * it once, loudly, naming the knob that actually fixes it. */
+/* #687: a resident tensor that fails to upload falls back to CPU silently. The
+ * per-tensor line below is easy to lose in a busy log, and the end state -- GPU fully
+ * allocated, ~0% useful -- reads as healthy. Count them and say it once, loudly,
+ * naming the knob that actually fixes it.
+ *
+ * Since #767 the fallback is per-width rather than permanent (see cuda_fail_s), so a
+ * tensor counted here may recover on its own at a narrower S. The count is still worth
+ * shouting about: it means the tier left too little VRAM for the prompt being run. */
static int g_cuda_disabled_n;
static void cuda_disabled_note(void){
enum { CUDA_DISABLED_LOUD_AT = 8 };
if(++g_cuda_disabled_n != CUDA_DISABLED_LOUD_AT) return;
fprintf(stderr,
- "[CUDA] ***** %d resident tensors have been disabled and moved to CPU *****\n"
- "[CUDA] The GPU still holds its expert tier, but the dense/attention path is now\n"
- "[CUDA] on CPU: this run will be at or below CPU speed while the card stays\n"
- "[CUDA] allocated. The usual cause is the expert tier claiming all VRAM and\n"
- "[CUDA] leaving nothing for the lazily-uploaded resident tensors.\n"
+ "[CUDA] ***** %d resident tensors have fallen back to CPU *****\n"
+ "[CUDA] The GPU still holds its expert tier, but the dense/attention path is\n"
+ "[CUDA] running on CPU at this prompt width: expect at or below CPU speed while\n"
+ "[CUDA] the card stays allocated. They are retried at narrower S, so decode may\n"
+ "[CUDA] recover even when prefill does not. The usual cause is the expert tier\n"
+ "[CUDA] claiming all VRAM and leaving nothing for the lazily-uploaded tensors.\n"
"[CUDA] Fix: set an explicit CUDA_EXPERT_GB below the auto value (leave room\n"
"[CUDA] per device for dense + attention + workspace). See issue #687.\n",
CUDA_DISABLED_LOUD_AT);
@@ -600,8 +613,8 @@ static void cuda_stats_print(void){
/* #687: say it again at the end -- by now the per-tensor lines are thousands of
* log lines back, and this is the number that explains a CPU-speed "GPU" run. */
if(g_cuda_disabled_n) fprintf(stderr,
- "[CUDA] %d tensors ran on CPU after failed uploads: this run did NOT use the GPU for "
- "them. Lower CUDA_EXPERT_GB (#687).\n", g_cuda_disabled_n);
+ "[CUDA] %d tensors fell back to CPU after failed uploads at least once. Lower "
+ "CUDA_EXPERT_GB to leave room for them (#687, #767).\n", g_cuda_disabled_n);
if(g_cuda_ndev>1) for(int i=0;i<g_cuda_ndev;i++){
coli_cuda_stats(g_cuda_devices[i],&n,&b);
fprintf(stderr,"[CUDA] device %d: %zu tensors, %.2f GB\n",g_cuda_devices[i],n,b/1e9);
@@ -879,15 +892,18 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot)
* cuda_boot): quant_matmul dispatches it by format, deriving the 128x128
* block-scale geometry from the dims alone. Without the LUT (old DLL) it is
* excluded up front — same silent-and-immediate idiom as fmt=5. */
- if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && w->fmt!=5 &&
- (w->fmt!=8 || g_cuda_fp8_ready) && !omp_in_parallel()){
+ if(g_cuda_enabled && w->cuda_eligible && (!w->cuda_fail_s || S < w->cuda_fail_s) &&
+ w->fmt!=5 && (w->fmt!=8 || g_cuda_fp8_ready) && !omp_in_parallel()){
const void *weights = w->fmt==0 ? (const void*)w->qf
: (w->fmt==1||w->fmt==8) ? (const void*)w->q8 : (const void*)w->q4;
- if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device,w->gs)) return;
- w->cuda_failed=1;
+ if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device,w->gs)){
+ w->cuda_fail_s=0; /* it fits again: forget the width that did not */
+ return;
+ }
+ w->cuda_fail_s = S;
if(g_cuda_disabled_n < 8) /* keep the detail for the first few, then the summary */
- fprintf(stderr,"[CUDA] tensor [%d,%d] on device %d disabled after an error; falling back to CPU\n",
- w->O,w->I,w->cuda_device);
+ fprintf(stderr,"[CUDA] tensor [%d,%d] on device %d fell back to CPU at S=%d; "
+ "narrower calls will still try the GPU\n", w->O,w->I,w->cuda_device,S);
cuda_disabled_note();
}
#endif