Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda: reuse decode activations across attention projections

ZacharyZcRca05b77b91da
diff --git a/c/colibri.c b/c/colibri.c
index 65acbf30..6cebfe0e 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -2583,6 +2583,32 @@ done:
free(chost); /* gli scratch device restano al contesto */
return ok;
}
+
+/* Decode-scale prefix only: upload x once and reuse it for q_a and kv_a. Keep
+ * the stock CPU RMSNorm between q_a and q_b so greedy output stays exact, then
+ * return Q/comp to the stock attention path. */
+static int attn_pipe_prefix(Model *m,Layer *l,const float *x,int S,float *QR,float *Q,float *comp){
+ Cfg *c=&m->c; int D=c->hidden,ql=c->q_lora,kvl=c->kv_lora,R=c->qk_rope;
+ int dev=l->q_a.cuda_device;
+ if(S<1||S>4||dev<0||l->q_b.cuda_device!=dev||l->kv_a.cuda_device!=dev) return 0;
+ size_t xb=(size_t)S*D*4,qrb=(size_t)S*ql*4;
+ size_t qb=(size_t)S*c->n_heads*c->qk_head*4,cb=(size_t)S*(kvl+R)*4;
+ float *xd=coli_cuda_pipe_scratch(dev,0,xb);
+ float *qrd=coli_cuda_pipe_scratch(dev,1,qrb);
+ float *qd=coli_cuda_pipe_scratch(dev,2,qb);
+ float *cd=coli_cuda_pipe_scratch(dev,3,cb);
+ if(!xd||!qrd||!qd||!cd) return 0;
+ if(!coli_cuda_pipe_upload(dev,xd,x,xb)||
+ !coli_cuda_pipe_gemm(l->q_a.cuda,qrd,xd,S)||
+ !coli_cuda_pipe_download(dev,qrd,QR,qrb)) return 0;
+ for(int s=0;s<S;s++) rmsnorm(QR+(int64_t)s*ql,QR+(int64_t)s*ql,l->q_a_ln,ql,c->eps);
+ if(!coli_cuda_pipe_upload(dev,qrd,QR,qrb)||
+ !coli_cuda_pipe_gemm(l->q_b.cuda,qd,qrd,S)||
+ !coli_cuda_pipe_gemm(l->kv_a.cuda,cd,xd,S)||
+ !coli_cuda_pipe_download(dev,qd,Q,qb)||
+ !coli_cuda_pipe_download(dev,cd,comp,cb)) return 0;
+ return 1;
+}
#endif
static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pos_base,
@@ -2641,7 +2667,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
* batch-union. matmul_qt_ex(...,0) keeps them on the EXACT int4 kernel: letting S>1 pull
* them into IDOT is much faster but costs ~12% perplexity (measured). Batching alone is
* bit-identical to upstream; the kernel switch is not. */
- int pipe_done=0;
+ int pipe_done=0,prefix_done=0;
#ifdef COLI_CUDA
if(g_cuda_pipe&&!kvs&&S>=8&&layer<c->n_layers&&g_cuda_enabled&&c->kv_lora<=512&&
!(m->has_dsa&&pos_base+S>c->index_topk)&&
@@ -2650,8 +2676,14 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
qt_cuda_upload(&l->q_a)&&qt_cuda_upload(&l->q_b)&&qt_cuda_upload(&l->kv_a)&&
qt_cuda_upload(&l->kv_b)&&qt_cuda_upload(&l->o))
pipe_done=attn_pipe_prefill(m,l,layer,x,0,S,pos_base,out,NULL);
+ if(!pipe_done&&getenv("COLI_CUDA_ATTN_PREFIX")&&atoi(getenv("COLI_CUDA_ATTN_PREFIX"))&&
+ !kvs&&S<=4&&g_cuda_enabled&&
+ (!m->has_dsa||pos_base+S<=c->index_topk)&&
+ l->q_a.cuda_eligible&&l->q_b.cuda_eligible&&l->kv_a.cuda_eligible&&
+ qt_cuda_upload(&l->q_a)&&qt_cuda_upload(&l->q_b)&&qt_cuda_upload(&l->kv_a))
+ prefix_done=attn_pipe_prefix(m,l,x,S,QR,Q,comp);
#endif
- if(!pipe_done){
+ if(!pipe_done&&!prefix_done){
int vk_qp=0; (void)vk_qp;
#ifdef COLI_VULKAN
/* Single-submit q-prep chain: [q_a+kv_a] -> rmsnorm(q latent, on-GPU) -> q_b.
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 49c87ca8..8d06ef0a 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -110,6 +110,7 @@ See [docs/vulkan.md](vulkan.md). On multi-core boxes also set `COLI_NO_OMP_TUNE=
| `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. |
| `CUDA_RELEASE_HOST` | auto (`1` if >1 device) | Release host-side copies after upload. |
| `COLI_CUDA_ATTN` | off | Run S≤4 attention on the GPU. |
+| `COLI_CUDA_ATTN_PREFIX` | off | Reuse one uploaded decode activation across `q_a` and `kv_a` while preserving the stock CPU RMSNorm path. |
| `COLI_CUDA_ATTN_SHARD` | off | `=1` splits KV-b heads across devices during attention load (multi-GPU). |
| `COLI_CUDA_PROFILE` | off | Emit CUDA timing. |
| `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. |