Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Merge dev: fallback diagnostics on the heap-allocated batch of #1686
JustVuggc07f62c6ea41
diff --cc c/qwen38_core.hindex f68cb4f8,eecf200b..8c658e5a--- a/c/qwen38_core.h+++ b/c/qwen38_core.h@@@ -110,16 -111,6 +111,15 @@@ typedef structint ready; /* 0 unknown, 1 resident, -1 incompatible */} Q38ExpertScaleCache;+typedef enum {+ Q38_EXPERT_BATCH_FALLBACK_NONE = 0,+ Q38_EXPERT_BATCH_FALLBACK_DISABLED,- Q38_EXPERT_BATCH_FALLBACK_TOO_MANY,+ Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY,+ Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK,+ Q38_EXPERT_BATCH_FALLBACK_DUPLICATE,+ Q38_EXPERT_BATCH_FALLBACK_LAYOUT,+} Q38ExpertBatchFallback;+typedef struct {Cfg c;shards S;@@@ -148,9 -140,9 +149,10 @@@int ple_history_len;int range_begin, range_end;int native_fp8, native_bf16, expert_prefetch, expert_parallel_reads;+ Q38ExpertBatchFallback expert_batch_fallback;int prefill_batch;uint64_t resident_weight_bytes;+ int trunk_table_built; /* q38_trunk_offer_all ran for this load (the table is process-wide, the model is not) */double dense_load_s;/* vision. `vis_map` mappa la posizione ASSOLUTA nella sequenza alla riga di* `vis_rows`, oppure -1. Assoluta e non relativa al chunk: il prefill arriva@@@ -1316,71 -1378,38 +1388,77 @@@ typedef structst_tensor *weight[3];} Q38ExpertLoadJob;+ /* Reserve an entire routed demand set on the main thread, fill distinct slots+ * in parallel, then publish every cache index on the main thread. Demand-set+ * residents are protected from victim selection, so no worker can overwrite a+ * slot another selected expert will consume. Smaller caches and heterogeneous+ * layouts retain the serial LRU path. */++/* Says once per model why the parallel read path is not taken, so a slow++ * prefill on a small cache or a converted container is not a mystery. */+static int q38_expert_batch_fallback(Model *m,Q38ExpertBatchFallback reason,+ int layer,int first,int second) {+ if(m->expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_NONE)return 0;+ m->expert_batch_fallback=reason;+ fprintf(stderr,"[qwen38 expert I/O] parallel reads unavailable: ");+ switch(reason){+ case Q38_EXPERT_BATCH_FALLBACK_DISABLED:+ fprintf(stderr,"Q38_EXPERT_PARALLEL_READS=0");break;- case Q38_EXPERT_BATCH_FALLBACK_TOO_MANY:- fprintf(stderr,"route needs %d experts, above the batch limit %d",first,second);break;+ case Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY:+ fprintf(stderr,"cache holds %d experts/layer but the route needs %d; "+ "lower --ctx/Q38_MAXT or raise --ram",first,second);break;+ case Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK:+ fprintf(stderr,"layer %d has no compatible resident FP8 scale bank",layer);break;+ case Q38_EXPERT_BATCH_FALLBACK_DUPLICATE:+ fprintf(stderr,"layer %d route repeats expert %d",layer,first);break;+ case Q38_EXPERT_BATCH_FALLBACK_LAYOUT:+ fprintf(stderr,"layer %d expert %d is not native block-FP8",layer,first);break;+ default:+ fprintf(stderr,"unknown reason");break;+ }+ fprintf(stderr,"; using serial expert reads\n");+ return 0;+}+- /* Reserve an entire routed demand set on the main thread, fill distinct slots- * in parallel, then publish every cache index on the main thread. Demand-set- * residents are protected from victim selection, so no worker can overwrite a- * slot another selected expert will consume. Smaller caches and heterogeneous- * layouts retain the serial LRU path. */static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,Slot **selected) {- if(!m->expert_parallel_reads||!experts||!selected||count<2||- count>m->cache[layer].cap)return 0;+ if(!experts||!selected||count<2)return 0;+ if(!m->expert_parallel_reads)- return q38_expert_batch_fallback(- m,Q38_EXPERT_BATCH_FALLBACK_DISABLED,layer,0,0);- if(count>Q38_MAX_TOPK)- return q38_expert_batch_fallback(- m,Q38_EXPERT_BATCH_FALLBACK_TOO_MANY,layer,count,Q38_MAX_TOPK);++ return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_DISABLED,layer,0,0);LCache *cache=&m->cache[layer];- if(cache->cap<count)- return q38_expert_batch_fallback(- m,Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY,layer,cache->cap,count);- if(!q38_prepare_expert_scale_bank(m,layer))return 0;++ if(count>cache->cap)++ return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY,++ layer,cache->cap,count);+ if(!q38_prepare_expert_scale_bank(m,layer))- return q38_expert_batch_fallback(- m,Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK,layer,0,0);- Q38ExpertLoadJob jobs[Q38_MAX_TOPK];++ return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK,layer,0,0);+ /* The demand set is no longer bounded by the decode top-k: the MoE prefill+ * hands over the whole chunk union (up to the cache cap) so its loads run+ * one OMP wave instead of serial groups of Q38_MAX_TOPK. Load grouping+ * never touches FP order: routed outputs are written per assignment and+ * the per-position expert sum follows the router order, so a bigger wave+ * only changes WHICH slots serve the reads, not the arithmetic. */+ Q38ExpertLoadJob *jobs=malloc((size_t)count*sizeof(*jobs));+ if(!jobs)return 0;for(int index=0;index<count;index++){int expert=experts[index];- if(expert<0||expert>=m->c.experts)return 0;+ if(expert<0||expert>=m->c.experts){free(jobs);return 0;}q38_ehit_mark(m,layer,expert);for(int previous=0;previous<index;previous++)- if(experts[previous]==expert)- return q38_expert_batch_fallback(- m,Q38_EXPERT_BATCH_FALLBACK_DUPLICATE,layer,expert,0);- if(experts[previous]==expert){free(jobs);return 0;}++ if(experts[previous]==expert){++ free(jobs);++ return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_DUPLICATE,++ layer,expert,0);++ }int slot_index=cache->by_expert[expert];if(slot_index>=0){- if(slot_index>=cache->n||cache->slots[slot_index].eid!=expert)return 0;+ if(slot_index>=cache->n||cache->slots[slot_index].eid!=expert){free(jobs);return 0;}continue;}st_tensor *weight[3];- if(!q38_native_fp8_expert_tensors(m,layer,expert,weight))- return q38_expert_batch_fallback(- m,Q38_EXPERT_BATCH_FALLBACK_LAYOUT,layer,expert,0);- if(!q38_native_fp8_expert_tensors(m,layer,expert,weight)){free(jobs);return 0;}++ if(!q38_native_fp8_expert_tensors(m,layer,expert,weight)){++ free(jobs);++ return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_LAYOUT,++ layer,expert,0);++ }}unsigned char *protected_slots=(unsigned char*)calloc((size_t)cache->cap,1);if(!protected_slots){fprintf(stderr,"OOM expert batch reservations\n");exit(1);}diff --cc c/tests/test_qwen38_native_weights.cindex 3e08f209,52eb5169..078c6d3f--- a/c/tests/test_qwen38_native_weights.c+++ b/c/tests/test_qwen38_native_weights.c@@@ -403,73 -402,6 +404,74 @@@ cleanupdestroy_fixture_model(&model);return result;}+static int check_parallel_fallback_reasons(const char *directory){+ int ids[Q38_MAX_TOPK+1]={0};Slot *selected[Q38_MAX_TOPK+1]={0};+ Model model;++ if(init_fixture_model(&model,directory,1))return -1;+ model.expert_parallel_reads=0;+ if(q38_expert_get_batch(&model,0,ids,2,selected)||+ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_DISABLED){+ destroy_fixture_model(&model);return -1;+ }+ model.expert_parallel_reads=1;+ if(q38_expert_get_batch(&model,0,ids,2,selected)||+ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_DISABLED){+ destroy_fixture_model(&model);return -1;+ }+ destroy_fixture_model(&model);++ if(init_fixture_model(&model,directory,1))return -1;+ model.expert_parallel_reads=1;- if(q38_expert_get_batch(&model,0,ids,Q38_MAX_TOPK+1,selected)||- model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_TOO_MANY){++ /* a route wider than the per-layer cache (the #1686 shape: no top-k ceiling) */++ if(q38_expert_get_batch(&model,0,ids,2,selected)||++ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY){+ destroy_fixture_model(&model);return -1;+ }+ destroy_fixture_model(&model);++ if(init_fixture_model(&model,directory,0)||q38_segment_cache_resize(&model,2)){+ destroy_fixture_model(&model);return -1;+ }+ model.expert_parallel_reads=1;+ if(q38_expert_get_batch(&model,0,ids,2,selected)||+ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK){+ destroy_fixture_model(&model);return -1;+ }+ destroy_fixture_model(&model);++ if(init_fixture_model(&model,directory,1)||q38_segment_cache_resize(&model,2)){+ destroy_fixture_model(&model);return -1;+ }+ model.expert_parallel_reads=1;+ if(q38_expert_get_batch(&model,0,ids,2,selected)||+ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_DUPLICATE){+ destroy_fixture_model(&model);return -1;+ }+ destroy_fixture_model(&model);++ if(init_fixture_model(&model,directory,1)||q38_segment_cache_resize(&model,2)){+ destroy_fixture_model(&model);return -1;+ }+ model.expert_parallel_reads=1;+ if(!q38_prepare_expert_scale_bank(&model,0)){+ destroy_fixture_model(&model);return -1;+ }+ char name[320];+ q38_name(&model,name,sizeof name,0,"mlp.experts.1.gate_proj.weight");+ st_tensor *weight=st_find(&model.S,name);+ if(!weight){destroy_fixture_model(&model);return -1;}+ int original_dtype=weight->dtype;weight->dtype=0;+ int layout_result=q38_expert_get_batch(&model,0,(int[]){0,1},2,selected);+ weight->dtype=original_dtype;+ if(layout_result||+ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_LAYOUT){+ destroy_fixture_model(&model);return -1;+ }+ destroy_fixture_model(&model);+ return 0;+}+static int check_malformed_scale_metadata(const char *directory){Model model;if(init_fixture_model(&model,directory,1))return -1;char name[320];