Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
cuda: reuse decode activations across attention projections
ZacharyZcRca05b77b91da
diff --git a/c/colibri.c b/c/colibri.cindex 65acbf30..6cebfe0e 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -2583,6 +2583,32 @@ done:free(chost); /* gli scratch device restano al contesto */return ok;}++/* Decode-scale prefix only: upload x once and reuse it for q_a and kv_a. Keep+ * the stock CPU RMSNorm between q_a and q_b so greedy output stays exact, then+ * return Q/comp to the stock attention path. */+static int attn_pipe_prefix(Model *m,Layer *l,const float *x,int S,float *QR,float *Q,float *comp){+ Cfg *c=&m->c; int D=c->hidden,ql=c->q_lora,kvl=c->kv_lora,R=c->qk_rope;+ int dev=l->q_a.cuda_device;+ if(S<1||S>4||dev<0||l->q_b.cuda_device!=dev||l->kv_a.cuda_device!=dev) return 0;+ size_t xb=(size_t)S*D*4,qrb=(size_t)S*ql*4;+ size_t qb=(size_t)S*c->n_heads*c->qk_head*4,cb=(size_t)S*(kvl+R)*4;+ float *xd=coli_cuda_pipe_scratch(dev,0,xb);+ float *qrd=coli_cuda_pipe_scratch(dev,1,qrb);+ float *qd=coli_cuda_pipe_scratch(dev,2,qb);+ float *cd=coli_cuda_pipe_scratch(dev,3,cb);+ if(!xd||!qrd||!qd||!cd) return 0;+ if(!coli_cuda_pipe_upload(dev,xd,x,xb)||+ !coli_cuda_pipe_gemm(l->q_a.cuda,qrd,xd,S)||+ !coli_cuda_pipe_download(dev,qrd,QR,qrb)) return 0;+ for(int s=0;s<S;s++) rmsnorm(QR+(int64_t)s*ql,QR+(int64_t)s*ql,l->q_a_ln,ql,c->eps);+ if(!coli_cuda_pipe_upload(dev,qrd,QR,qrb)||+ !coli_cuda_pipe_gemm(l->q_b.cuda,qd,qrd,S)||+ !coli_cuda_pipe_gemm(l->kv_a.cuda,cd,xd,S)||+ !coli_cuda_pipe_download(dev,qd,Q,qb)||+ !coli_cuda_pipe_download(dev,cd,comp,cb)) return 0;+ return 1;+}#endifstatic void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pos_base,@@ -2641,7 +2667,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p* batch-union. matmul_qt_ex(...,0) keeps them on the EXACT int4 kernel: letting S>1 pull* them into IDOT is much faster but costs ~12% perplexity (measured). Batching alone is* bit-identical to upstream; the kernel switch is not. */- int pipe_done=0;+ int pipe_done=0,prefix_done=0;#ifdef COLI_CUDAif(g_cuda_pipe&&!kvs&&S>=8&&layer<c->n_layers&&g_cuda_enabled&&c->kv_lora<=512&&!(m->has_dsa&&pos_base+S>c->index_topk)&&@@ -2650,8 +2676,14 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pqt_cuda_upload(&l->q_a)&&qt_cuda_upload(&l->q_b)&&qt_cuda_upload(&l->kv_a)&&qt_cuda_upload(&l->kv_b)&&qt_cuda_upload(&l->o))pipe_done=attn_pipe_prefill(m,l,layer,x,0,S,pos_base,out,NULL);+ if(!pipe_done&&getenv("COLI_CUDA_ATTN_PREFIX")&&atoi(getenv("COLI_CUDA_ATTN_PREFIX"))&&+ !kvs&&S<=4&&g_cuda_enabled&&+ (!m->has_dsa||pos_base+S<=c->index_topk)&&+ l->q_a.cuda_eligible&&l->q_b.cuda_eligible&&l->kv_a.cuda_eligible&&+ qt_cuda_upload(&l->q_a)&&qt_cuda_upload(&l->q_b)&&qt_cuda_upload(&l->kv_a))+ prefix_done=attn_pipe_prefix(m,l,x,S,QR,Q,comp);#endif- if(!pipe_done){+ if(!pipe_done&&!prefix_done){int vk_qp=0; (void)vk_qp;#ifdef COLI_VULKAN/* Single-submit q-prep chain: [q_a+kv_a] -> rmsnorm(q latent, on-GPU) -> q_b.diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 49c87ca8..8d06ef0a 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -110,6 +110,7 @@ See [docs/vulkan.md](vulkan.md). On multi-core boxes also set `COLI_NO_OMP_TUNE=| `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. || `CUDA_RELEASE_HOST` | auto (`1` if >1 device) | Release host-side copies after upload. || `COLI_CUDA_ATTN` | off | Run S≤4 attention on the GPU. |+| `COLI_CUDA_ATTN_PREFIX` | off | Reuse one uploaded decode activation across `q_a` and `kv_a` while preserving the stock CPU RMSNorm path. || `COLI_CUDA_ATTN_SHARD` | off | `=1` splits KV-b heads across devices during attention load (multi-GPU). || `COLI_CUDA_PROFILE` | off | Emit CUDA timing. || `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. |