Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge dev: fallback diagnostics on the heap-allocated batch of #1686

JustVuggc07f62c6ea41
diff --cc c/qwen38_core.h
index f68cb4f8,eecf200b..8c658e5a
--- a/c/qwen38_core.h
+++ b/c/qwen38_core.h
@@@ -110,16 -111,6 +111,15 @@@ typedef struct
int ready; /* 0 unknown, 1 resident, -1 incompatible */
} Q38ExpertScaleCache;
+typedef enum {
+ Q38_EXPERT_BATCH_FALLBACK_NONE = 0,
+ Q38_EXPERT_BATCH_FALLBACK_DISABLED,
- Q38_EXPERT_BATCH_FALLBACK_TOO_MANY,
+ Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY,
+ Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK,
+ Q38_EXPERT_BATCH_FALLBACK_DUPLICATE,
+ Q38_EXPERT_BATCH_FALLBACK_LAYOUT,
+} Q38ExpertBatchFallback;
+
typedef struct {
Cfg c;
shards S;
@@@ -148,9 -140,9 +149,10 @@@
int ple_history_len;
int range_begin, range_end;
int native_fp8, native_bf16, expert_prefetch, expert_parallel_reads;
+ Q38ExpertBatchFallback expert_batch_fallback;
int prefill_batch;
uint64_t resident_weight_bytes;
+ int trunk_table_built; /* q38_trunk_offer_all ran for this load (the table is process-wide, the model is not) */
double dense_load_s;
/* vision. `vis_map` mappa la posizione ASSOLUTA nella sequenza alla riga di
* `vis_rows`, oppure -1. Assoluta e non relativa al chunk: il prefill arriva
@@@ -1316,71 -1378,38 +1388,77 @@@ typedef struct
st_tensor *weight[3];
} Q38ExpertLoadJob;
+ /* Reserve an entire routed demand set on the main thread, fill distinct slots
+ * in parallel, then publish every cache index on the main thread. Demand-set
+ * residents are protected from victim selection, so no worker can overwrite a
+ * slot another selected expert will consume. Smaller caches and heterogeneous
+ * layouts retain the serial LRU path. */
++/* Says once per model why the parallel read path is not taken, so a slow
++ * prefill on a small cache or a converted container is not a mystery. */
+static int q38_expert_batch_fallback(Model *m,Q38ExpertBatchFallback reason,
+ int layer,int first,int second) {
+ if(m->expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_NONE)return 0;
+ m->expert_batch_fallback=reason;
+ fprintf(stderr,"[qwen38 expert I/O] parallel reads unavailable: ");
+ switch(reason){
+ case Q38_EXPERT_BATCH_FALLBACK_DISABLED:
+ fprintf(stderr,"Q38_EXPERT_PARALLEL_READS=0");break;
- case Q38_EXPERT_BATCH_FALLBACK_TOO_MANY:
- fprintf(stderr,"route needs %d experts, above the batch limit %d",first,second);break;
+ case Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY:
+ fprintf(stderr,"cache holds %d experts/layer but the route needs %d; "
+ "lower --ctx/Q38_MAXT or raise --ram",first,second);break;
+ case Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK:
+ fprintf(stderr,"layer %d has no compatible resident FP8 scale bank",layer);break;
+ case Q38_EXPERT_BATCH_FALLBACK_DUPLICATE:
+ fprintf(stderr,"layer %d route repeats expert %d",layer,first);break;
+ case Q38_EXPERT_BATCH_FALLBACK_LAYOUT:
+ fprintf(stderr,"layer %d expert %d is not native block-FP8",layer,first);break;
+ default:
+ fprintf(stderr,"unknown reason");break;
+ }
+ fprintf(stderr,"; using serial expert reads\n");
+ return 0;
+}
+
- /* Reserve an entire routed demand set on the main thread, fill distinct slots
- * in parallel, then publish every cache index on the main thread. Demand-set
- * residents are protected from victim selection, so no worker can overwrite a
- * slot another selected expert will consume. Smaller caches and heterogeneous
- * layouts retain the serial LRU path. */
static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,
Slot **selected) {
- if(!m->expert_parallel_reads||!experts||!selected||count<2||
- count>m->cache[layer].cap)return 0;
+ if(!experts||!selected||count<2)return 0;
+ if(!m->expert_parallel_reads)
- return q38_expert_batch_fallback(
- m,Q38_EXPERT_BATCH_FALLBACK_DISABLED,layer,0,0);
- if(count>Q38_MAX_TOPK)
- return q38_expert_batch_fallback(
- m,Q38_EXPERT_BATCH_FALLBACK_TOO_MANY,layer,count,Q38_MAX_TOPK);
++ return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_DISABLED,layer,0,0);
LCache *cache=&m->cache[layer];
- if(cache->cap<count)
- return q38_expert_batch_fallback(
- m,Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY,layer,cache->cap,count);
- if(!q38_prepare_expert_scale_bank(m,layer))return 0;
++ if(count>cache->cap)
++ return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY,
++ layer,cache->cap,count);
+ if(!q38_prepare_expert_scale_bank(m,layer))
- return q38_expert_batch_fallback(
- m,Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK,layer,0,0);
- Q38ExpertLoadJob jobs[Q38_MAX_TOPK];
++ return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK,layer,0,0);
+ /* The demand set is no longer bounded by the decode top-k: the MoE prefill
+ * hands over the whole chunk union (up to the cache cap) so its loads run
+ * one OMP wave instead of serial groups of Q38_MAX_TOPK. Load grouping
+ * never touches FP order: routed outputs are written per assignment and
+ * the per-position expert sum follows the router order, so a bigger wave
+ * only changes WHICH slots serve the reads, not the arithmetic. */
+ Q38ExpertLoadJob *jobs=malloc((size_t)count*sizeof(*jobs));
+ if(!jobs)return 0;
for(int index=0;index<count;index++){
int expert=experts[index];
- if(expert<0||expert>=m->c.experts)return 0;
+ if(expert<0||expert>=m->c.experts){free(jobs);return 0;}
q38_ehit_mark(m,layer,expert);
for(int previous=0;previous<index;previous++)
- if(experts[previous]==expert)
- return q38_expert_batch_fallback(
- m,Q38_EXPERT_BATCH_FALLBACK_DUPLICATE,layer,expert,0);
- if(experts[previous]==expert){free(jobs);return 0;}
++ if(experts[previous]==expert){
++ free(jobs);
++ return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_DUPLICATE,
++ layer,expert,0);
++ }
int slot_index=cache->by_expert[expert];
if(slot_index>=0){
- if(slot_index>=cache->n||cache->slots[slot_index].eid!=expert)return 0;
+ if(slot_index>=cache->n||cache->slots[slot_index].eid!=expert){free(jobs);return 0;}
continue;
}
st_tensor *weight[3];
- if(!q38_native_fp8_expert_tensors(m,layer,expert,weight))
- return q38_expert_batch_fallback(
- m,Q38_EXPERT_BATCH_FALLBACK_LAYOUT,layer,expert,0);
- if(!q38_native_fp8_expert_tensors(m,layer,expert,weight)){free(jobs);return 0;}
++ if(!q38_native_fp8_expert_tensors(m,layer,expert,weight)){
++ free(jobs);
++ return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_LAYOUT,
++ layer,expert,0);
++ }
}
unsigned char *protected_slots=(unsigned char*)calloc((size_t)cache->cap,1);
if(!protected_slots){fprintf(stderr,"OOM expert batch reservations\n");exit(1);}
diff --cc c/tests/test_qwen38_native_weights.c
index 3e08f209,52eb5169..078c6d3f
--- a/c/tests/test_qwen38_native_weights.c
+++ b/c/tests/test_qwen38_native_weights.c
@@@ -403,73 -402,6 +404,74 @@@ cleanup
destroy_fixture_model(&model);return result;
}
+static int check_parallel_fallback_reasons(const char *directory){
+ int ids[Q38_MAX_TOPK+1]={0};Slot *selected[Q38_MAX_TOPK+1]={0};
+ Model model;
+
+ if(init_fixture_model(&model,directory,1))return -1;
+ model.expert_parallel_reads=0;
+ if(q38_expert_get_batch(&model,0,ids,2,selected)||
+ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_DISABLED){
+ destroy_fixture_model(&model);return -1;
+ }
+ model.expert_parallel_reads=1;
+ if(q38_expert_get_batch(&model,0,ids,2,selected)||
+ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_DISABLED){
+ destroy_fixture_model(&model);return -1;
+ }
+ destroy_fixture_model(&model);
+
+ if(init_fixture_model(&model,directory,1))return -1;
+ model.expert_parallel_reads=1;
- if(q38_expert_get_batch(&model,0,ids,Q38_MAX_TOPK+1,selected)||
- model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_TOO_MANY){
++ /* a route wider than the per-layer cache (the #1686 shape: no top-k ceiling) */
++ if(q38_expert_get_batch(&model,0,ids,2,selected)||
++ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY){
+ destroy_fixture_model(&model);return -1;
+ }
+ destroy_fixture_model(&model);
+
+ if(init_fixture_model(&model,directory,0)||q38_segment_cache_resize(&model,2)){
+ destroy_fixture_model(&model);return -1;
+ }
+ model.expert_parallel_reads=1;
+ if(q38_expert_get_batch(&model,0,ids,2,selected)||
+ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK){
+ destroy_fixture_model(&model);return -1;
+ }
+ destroy_fixture_model(&model);
+
+ if(init_fixture_model(&model,directory,1)||q38_segment_cache_resize(&model,2)){
+ destroy_fixture_model(&model);return -1;
+ }
+ model.expert_parallel_reads=1;
+ if(q38_expert_get_batch(&model,0,ids,2,selected)||
+ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_DUPLICATE){
+ destroy_fixture_model(&model);return -1;
+ }
+ destroy_fixture_model(&model);
+
+ if(init_fixture_model(&model,directory,1)||q38_segment_cache_resize(&model,2)){
+ destroy_fixture_model(&model);return -1;
+ }
+ model.expert_parallel_reads=1;
+ if(!q38_prepare_expert_scale_bank(&model,0)){
+ destroy_fixture_model(&model);return -1;
+ }
+ char name[320];
+ q38_name(&model,name,sizeof name,0,"mlp.experts.1.gate_proj.weight");
+ st_tensor *weight=st_find(&model.S,name);
+ if(!weight){destroy_fixture_model(&model);return -1;}
+ int original_dtype=weight->dtype;weight->dtype=0;
+ int layout_result=q38_expert_get_batch(&model,0,(int[]){0,1},2,selected);
+ weight->dtype=original_dtype;
+ if(layout_result||
+ model.expert_batch_fallback!=Q38_EXPERT_BATCH_FALLBACK_LAYOUT){
+ destroy_fixture_model(&model);return -1;
+ }
+ destroy_fixture_model(&model);
+ return 0;
+}
+
static int check_malformed_scale_metadata(const char *directory){
Model model;if(init_fixture_model(&model,directory,1))return -1;
char name[320];