Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen38: project the PLE keys and values per block of rows

JustVuggc66fdbae030b
diff --git a/c/qwen38_core.h b/c/qwen38_core.h
index 3026f336..9535d7b3 100644
--- a/c/qwen38_core.h
+++ b/c/qwen38_core.h
@@ -1939,44 +1939,63 @@ static void q38_ple_prefetch(Model *m,const int *ids,int S) {
m->ple_pref=buffer; m->ple_pref_rows=S;
}
+/* The key and value projections read only the token's n-gram embedding, so they
+ * run once for a block of rows: one S-row call (a device GEMM) instead of one
+ * GEMV per token. The embedding lookups walk the n-gram history and the
+ * convolution its ring, both in token order as before; every CPU kernel computes
+ * each row on its own, so a block gives the bits of the per-token calls. A
+ * projection the CUDA tier holds answers one row at a time (the tier serves
+ * S == 1, prefill rows would fall to the CPU), so with one the block is a row,
+ * as with Q38_PREFILL_BATCH=0. */
+static int q38_bounded_prefill_rows(int requested,uint64_t fixed,uint64_t per_row);
static void q38_ple(Model *m,const int *ids,int S,const float *hyper,float *out) {
double phase_started=now_s();
Cfg *c=&m->c; Layer *l=&m->L[c->ple_layer]; int H=c->hidden,C=c->hc_count,W=c->hc_width,E=c->ple_dim;
- float *emb=falloc(E),*keys=falloc(W),*value=falloc(H),*kn=falloc(W),*qn=falloc(W),*gated=falloc(W),*norm=falloc(W);
+ int B=!m->prefill_batch||l->ple_key.gpu||l->ple_value.gpu?1:
+ q38_bounded_prefill_rows(S,0,((uint64_t)E+(uint64_t)W+(uint64_t)H)*sizeof(float));
+ float *embs=falloc((int64_t)B*E),*keysb=falloc((int64_t)B*W),*valueb=falloc((int64_t)B*H);
+ float *kn=falloc(W),*qn=falloc(W),*gated=falloc(W),*norm=falloc(W);
int state_len=(c->ple_convk-1)*c->ngram_size; float *ring=m->PLE_conv_state;
- for(int s=0;s<S;s++){
- int64_t p1=m->ple_history_len>=1?m->ple_history[m->ple_history_len-1]:c->eos_id;
- int64_t p2=m->ple_history_len>=2?m->ple_history[m->ple_history_len-2]:c->eos_id;
- if(m->ple_pref&&s<m->ple_pref_rows){
- /* gia' in memoria: le ha portate q38_ple_prefetch mentre i primi
- * layer calcolavano */
- memcpy(emb,m->ple_pref+(int64_t)s*c->ngram_heads*c->ngram_head_dim,
- (size_t)c->ngram_heads*c->ngram_head_dim*sizeof(float));
- } else for(int h=0;h<c->ngram_heads;h++){
- int ng=h<c->heads_per_ngram?2:3; int64_t row=q38_hash_row(m,h,ng,ids[s],p1,p2);
- q38_ple_row(m,row,emb+(int64_t)h*c->ngram_head_dim);
- }
- q38_dense_matmul(m,keys,emb,&l->ple_key,1,E,W);q38_dense_matmul(m,value,emb,&l->ple_value,1,E,H);
- for(int b=0;b<C;b++){
- q38_rms0(kn+(int64_t)b*H,keys+(int64_t)b*H,l->ple_norm_key+(int64_t)b*H,H,c->eps);
- q38_rms0(qn+(int64_t)b*H,hyper+(int64_t)s*W+(int64_t)b*H,l->ple_norm_query+(int64_t)b*H,H,c->eps);
- float dot=0.f;for(int d=0;d<H;d++)dot+=kn[(int64_t)b*H+d]*qn[(int64_t)b*H+d];dot/=sqrtf((float)H);
- float shaped=copysignf(sqrtf(fmaxf(fabsf(dot),1e-6f)),dot),g=q38_sigmoid(shaped);
- for(int d=0;d<H;d++)gated[(int64_t)b*H+d]=g*value[d];
- q38_rms0(norm+(int64_t)b*H,gated+(int64_t)b*H,l->ple_norm_conv+(int64_t)b*H,H,c->eps);
+ for(int base=0;base<S;base+=B){
+ int rows=S-base<B?S-base:B;
+ for(int r=0;r<rows;r++){
+ int s=base+r; float *emb=embs+(int64_t)r*E;
+ int64_t p1=m->ple_history_len>=1?m->ple_history[m->ple_history_len-1]:c->eos_id;
+ int64_t p2=m->ple_history_len>=2?m->ple_history[m->ple_history_len-2]:c->eos_id;
+ if(m->ple_pref&&s<m->ple_pref_rows){
+ /* gia' in memoria: le ha portate q38_ple_prefetch mentre i primi
+ * layer calcolavano */
+ memcpy(emb,m->ple_pref+(int64_t)s*c->ngram_heads*c->ngram_head_dim,
+ (size_t)c->ngram_heads*c->ngram_head_dim*sizeof(float));
+ } else for(int h=0;h<c->ngram_heads;h++){
+ int ng=h<c->heads_per_ngram?2:3; int64_t row=q38_hash_row(m,h,ng,ids[s],p1,p2);
+ q38_ple_row(m,row,emb+(int64_t)h*c->ngram_head_dim);
+ }
+ if(ids[s]==c->eos_id)m->ple_history_len=0;
+ else if(m->ple_history_len==0){m->ple_history[0]=ids[s];m->ple_history_len=1;}
+ else if(m->ple_history_len==1){m->ple_history[1]=ids[s];m->ple_history_len=2;}
+ else {m->ple_history[0]=m->ple_history[1];m->ple_history[1]=ids[s];}
}
- for(int d=0;d<W;d++){
- float a=l->ple_conv[(int64_t)d*c->ple_convk+c->ple_convk-1]*norm[d];
- for(int k=0;k<c->ple_convk-1;k++)a+=l->ple_conv[(int64_t)d*c->ple_convk+k]*ring[(int64_t)d*state_len+k*c->ngram_size];
- out[(int64_t)s*W+d]=gated[d]+q38_silu(a);
- float *r=ring+(int64_t)d*state_len;for(int k=0;k<state_len-1;k++)r[k]=r[k+1];r[state_len-1]=norm[d];
+ q38_dense_matmul(m,keysb,embs,&l->ple_key,rows,E,W);q38_dense_matmul(m,valueb,embs,&l->ple_value,rows,E,H);
+ for(int r=0;r<rows;r++){
+ int s=base+r; const float *keys=keysb+(int64_t)r*W,*value=valueb+(int64_t)r*H;
+ for(int b=0;b<C;b++){
+ q38_rms0(kn+(int64_t)b*H,keys+(int64_t)b*H,l->ple_norm_key+(int64_t)b*H,H,c->eps);
+ q38_rms0(qn+(int64_t)b*H,hyper+(int64_t)s*W+(int64_t)b*H,l->ple_norm_query+(int64_t)b*H,H,c->eps);
+ float dot=0.f;for(int d=0;d<H;d++)dot+=kn[(int64_t)b*H+d]*qn[(int64_t)b*H+d];dot/=sqrtf((float)H);
+ float shaped=copysignf(sqrtf(fmaxf(fabsf(dot),1e-6f)),dot),g=q38_sigmoid(shaped);
+ for(int d=0;d<H;d++)gated[(int64_t)b*H+d]=g*value[d];
+ q38_rms0(norm+(int64_t)b*H,gated+(int64_t)b*H,l->ple_norm_conv+(int64_t)b*H,H,c->eps);
+ }
+ for(int d=0;d<W;d++){
+ float a=l->ple_conv[(int64_t)d*c->ple_convk+c->ple_convk-1]*norm[d];
+ for(int k=0;k<c->ple_convk-1;k++)a+=l->ple_conv[(int64_t)d*c->ple_convk+k]*ring[(int64_t)d*state_len+k*c->ngram_size];
+ out[(int64_t)s*W+d]=gated[d]+q38_silu(a);
+ float *rg=ring+(int64_t)d*state_len;for(int k=0;k<state_len-1;k++)rg[k]=rg[k+1];rg[state_len-1]=norm[d];
+ }
}
- if(ids[s]==c->eos_id)m->ple_history_len=0;
- else if(m->ple_history_len==0){m->ple_history[0]=ids[s];m->ple_history_len=1;}
- else if(m->ple_history_len==1){m->ple_history[1]=ids[s];m->ple_history_len=2;}
- else {m->ple_history[0]=m->ple_history[1];m->ple_history[1]=ids[s];}
}
- free(emb);free(keys);free(value);free(kn);free(qn);free(gated);free(norm);
+ free(embs);free(keysb);free(valueb);free(kn);free(qn);free(gated);free(norm);
q38_tm_add(m,Q38_TM_PLE,phase_started);
}