Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge dev: TEST_BINS is derived from build rules now (#733)

JustVugg6a4165ceae2b
diff --cc c/colibri.c
index 6ff1d9f5,8bf7cc4b..3a78b662
--- a/c/colibri.c
+++ b/c/colibri.c
@@@ -3073,33 -2608,34 +3098,59 @@@ done
free(chost); /* gli scratch device restano al contesto */
return ok;
}
+
+ /* Decode-scale prefix only: upload x once and reuse it for q_a and kv_a. Keep
+ * the stock CPU RMSNorm between q_a and q_b so greedy output stays exact, then
+ * return Q/comp to the stock attention path. */
+ static int attn_pipe_prefix(Model *m,Layer *l,const float *x,int S,float *QR,float *Q,float *comp){
+ Cfg *c=&m->c; int D=c->hidden,ql=c->q_lora,kvl=c->kv_lora,R=c->qk_rope;
+ int dev=l->q_a.cuda_device;
+ if(S<1||S>4||dev<0||l->q_b.cuda_device!=dev||l->kv_a.cuda_device!=dev) return 0;
+ size_t xb=(size_t)S*D*4,qrb=(size_t)S*ql*4;
+ size_t qb=(size_t)S*c->n_heads*c->qk_head*4,cb=(size_t)S*(kvl+R)*4;
+ float *xd=coli_cuda_pipe_scratch(dev,0,xb);
+ float *qrd=coli_cuda_pipe_scratch(dev,1,qrb);
+ float *qd=coli_cuda_pipe_scratch(dev,2,qb);
+ float *cd=coli_cuda_pipe_scratch(dev,3,cb);
+ if(!xd||!qrd||!qd||!cd) return 0;
+ if(!coli_cuda_pipe_upload(dev,xd,x,xb)||
+ !coli_cuda_pipe_gemm(l->q_a.cuda,qrd,xd,S)||
+ !coli_cuda_pipe_download(dev,qrd,QR,qrb)) return 0;
+ for(int s=0;s<S;s++) rmsnorm(QR+(int64_t)s*ql,QR+(int64_t)s*ql,l->q_a_ln,ql,c->eps);
+ if(!coli_cuda_pipe_upload(dev,qrd,QR,qrb)||
+ !coli_cuda_pipe_gemm(l->q_b.cuda,qd,qrd,S)||
+ !coli_cuda_pipe_gemm(l->kv_a.cuda,cd,xd,S)||
+ !coli_cuda_pipe_download(dev,qd,Q,qb)||
+ !coli_cuda_pipe_download(dev,cd,comp,cb)) return 0;
+ return 1;
+ }
#endif
+/* POSITIVE allowlist for the two fused Metal decode gates below
+ * (attention_rows / layer_forward_rows): only the mm_gemv shader's explicit
+ * INTEGER-format branches (fmt 1/2/3/4) may enter the fused path -- the
+ * exact set that was BOTH admitted by the old negative guard AND computed
+ * correctly. Anything else falls back to the CPU path (correct, slower).
+ * INVARIANT this predicate names: a NEGATIVE guard (the previous
+ * `fmt!=8` form) is fail-OPEN for every format it doesn't name --
+ * bind_gemv (backend_metal.mm) has no fmt allowlist of its own, and the
+ * mm_gemv shader's dispatch chain ends in an unconditional f32 fallback
+ * branch, so a resident fmt=5 or fmt=6 tensor slipping through would have
+ * its REAL weight bytes (they live in q4, which is exactly what the WP_()
+ * macro hands the kernel) silently misread as f32 garbage, with no error.
+ * This form is fail-CLOSED: a future format is excluded until someone
+ * proves it on the fused path and adds it here. fmt=3 (int2) is included
+ * because it was legal-and-correct here before this change (explicit
+ * mm_gemv branch, real q4 bytes) -- excluding it would be this change's
+ * only regression. fmt=0 (f32) stays excluded: it never actually fused --
+ * WP_() hands the shader q4, which is NULL for fmt=0 (weights live in qf)
+ * -- so admitting it would be dead permissiveness, not a preserved
+ * behavior. fmt=5/6 (misread hazard), fmt=8 (NULL-q4 variant), and every
+ * future format remain excluded.
+ * Defined unconditionally (not #ifdef COLI_METAL) so the CPU test build
+ * can unit-test the truth table (tests/test_fp8_load.c Part F). */
+static inline int metal_fused_fmt_ok(int fmt){ return fmt==1 || fmt==2 || fmt==3 || fmt==4; }
+
static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pos_base,
KVState *const *kvs, const int *positions, float *out){
Cfg *c=&m->c; int H=c->n_heads, D=c->hidden, qh=c->qk_head, vh=c->v_head;