Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(qwen38): overlap native FP8 expert loads
themorlock99deb09011cd
diff --git a/c/qwen38.c b/c/qwen38.cindex 7cf950d2..244c4ef4 100644--- a/c/qwen38.c+++ b/c/qwen38.c@@ -10,7 +10,8 @@* block-FP8 experts remain native in a bounded per-layer LRU; the 51B-parameter* PLE table remains on disk and only its sixteen 160-byte rows/token are read.*- * Runtime: SNAP=<dir>, optional Q38_MAXT, and argv+ * Runtime: SNAP=<dir>, optional Q38_MAXT, Q38_EXPERT_PREFETCH=0 and+ * Q38_EXPERT_PARALLEL_READS=0 A/B controls, and argv* `qwen38 <cache/layer> <reserved> [ref.json]`. The second positional remains* reserved so Colibri's common cache-aware launcher ABI stays stable.*/@@ -1625,10 +1626,11 @@ static int q38_segment_fused_matrix_bytes(Model *m,const st_tensor *tensor,* representative. Fused containers have one dtype/geometry for every expert. */static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,uint64_t *bytes_per_capacity,+ uint64_t *fixed_scale_bytes,unsigned *numeric_kinds){- if(!m||!bytes_per_capacity||!numeric_kinds||begin>=end||+ if(!m||!bytes_per_capacity||!fixed_scale_bytes||!numeric_kinds||begin>=end||end>(uint32_t)m->c.layers)return -1;- Cfg *c=&m->c;uint64_t range_bytes=0;unsigned kinds=0;+ Cfg *c=&m->c;uint64_t range_bytes=0,range_scales=0;unsigned kinds=0;for(uint32_t layer=begin;layer<end;layer++){char name[320];q38_name(m,name,sizeof name,(int)layer,"mlp.experts.gate_up_proj");@@ -1652,6 +1654,7 @@ static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,const char *projection[3]={"gate_proj","up_proj","down_proj"};int rows[3]={c->inter,c->inter,c->hidden};int cols[3]={c->hidden,c->hidden,c->inter};+ int all_native_fp8=m->native_fp8;for(int expert=0;expert<c->experts;expert++){uint64_t expert_bytes=0;for(int projection_index=0;projection_index<3;projection_index++){@@ -1661,6 +1664,8 @@ static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,projection[projection_index]);if(length<0||(size_t)length>=sizeof suffix)return -1;q38_name(m,name,sizeof name,(int)layer,suffix);+ st_tensor *weight=st_find(&m->S,name);+ if(!weight||weight->dtype!=4)all_native_fp8=0;uint64_t part;if(q38_segment_matrix_bytes(m,name,rows[projection_index],cols[projection_index],&part,&kinds)||@@ -1668,17 +1673,37 @@ static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,}if(expert_bytes>layer_bytes)layer_bytes=expert_bytes;}+ if(all_native_fp8){+ uint64_t scale_count=(uint64_t)fp8_nblk(c->inter)*+ (uint64_t)fp8_nblk(c->hidden);+ uint64_t per_slot_scales,layer_scale_bank;+ if(q38_u64_mul(3u,scale_count,&per_slot_scales)||+ q38_u64_mul(per_slot_scales,sizeof(float),&per_slot_scales)||+ layer_bytes<per_slot_scales||+ q38_u64_mul(per_slot_scales,(uint64_t)c->experts,+ &layer_scale_bank)||+ q38_u64_add(&range_scales,layer_scale_bank))return -1;+ /* Eligible checkpoints borrow normalized scales from one+ * per-layer bank, so only raw FP8 payload belongs to a slot.+ * Charging the full bank even for a repacked checkpoint that+ * later falls back is conservative and keeps tight Segment+ * memory limits safe. */+ layer_bytes-=per_slot_scales;+ }}if(!layer_bytes||q38_u64_add(&range_bytes,layer_bytes))return -1;}- *bytes_per_capacity=range_bytes;*numeric_kinds=kinds;return 0;+ *bytes_per_capacity=range_bytes;*fixed_scale_bytes=range_scales;+ *numeric_kinds=kinds;return 0;}-static int q38_segment_cache_capacity(uint64_t bytes_per_capacity,int experts,+static int q38_segment_cache_capacity(uint64_t bytes_per_capacity,+ uint64_t fixed_scale_bytes,int experts,uint64_t memory_limit,int *capacity){if(!bytes_per_capacity||experts<1||!capacity)return -1;if(!memory_limit){*capacity=1;return 0;}- uint64_t slots=memory_limit/bytes_per_capacity;+ if(memory_limit<=fixed_scale_bytes)return -1;+ uint64_t slots=(memory_limit-fixed_scale_bytes)/bytes_per_capacity;if(!slots)return -1;if(slots>(uint64_t)experts)slots=(uint64_t)experts;*capacity=(int)slots;return 0;@@ -1809,14 +1834,17 @@ static int qwen38_segment_engine_open(* cannot discard data or perturb LRU state. */model_init_range(&e->model,options->model_dir,1,8,(int)e->layer_begin,(int)e->layer_end,0,0);- uint64_t bytes_per_capacity=0;unsigned numeric_kinds=0;int cap=1;+ uint64_t bytes_per_capacity=0,fixed_scale_bytes=0;+ unsigned numeric_kinds=0;int cap=1;if(q38_segment_expert_layout(&e->model,e->layer_begin,e->layer_end,- &bytes_per_capacity,&numeric_kinds)){+ &bytes_per_capacity,&fixed_scale_bytes,+ &numeric_kinds)){q38_model_free(&e->model);pthread_mutex_destroy(&e->run_lock);free(e);return coli_segment_adapter_error(error,error_size,"Qwen3.8 Segment checkpoint has an unsupported expert layout");}- if(q38_segment_cache_capacity(bytes_per_capacity,e->model.c.experts,+ if(q38_segment_cache_capacity(bytes_per_capacity,fixed_scale_bytes,+ e->model.c.experts,options->memory_limit_bytes,&cap)){q38_model_free(&e->model);pthread_mutex_destroy(&e->run_lock);free(e);return coli_segment_adapter_error(error,error_size,diff --git a/c/qwen38_core.h b/c/qwen38_core.hindex e6bf42f1..67e09c61 100644--- a/c/qwen38_core.h+++ b/c/qwen38_core.h@@ -40,6 +40,7 @@ typedef struct {int rows, cols;int64_t elements, scale_count;Q38WeightKind kind;+ unsigned owns_data:1, owns_scales:1;} Q38Weight;typedef struct { float *norm; Q38Weight down, up, inject; } GatedResidual;@@ -84,9 +85,21 @@ typedef struct {float *ple_norm_conv, *ple_conv;} Layer;-typedef struct { int eid; Q38Weight gate, up, down; uint64_t used; } Slot;+typedef struct {+ int eid;+ Q38Weight gate, up, down;+ uint64_t used;+ void *fp8_slab;+ int64_t fp8_slab_bytes;+} Slot;typedef struct { Slot *slots; int *by_expert, n, cap; } LCache;+typedef struct {+ float *values; /* [expert][gate,up,down][block] */+ int64_t scale_count;+ int ready; /* 0 unknown, 1 resident, -1 incompatible */+} Q38ExpertScaleCache;+typedef struct {Cfg c;shards S;@@ -95,7 +108,11 @@ typedef struct {GatedResidual final_gr;Layer *L;LCache *cache;+ Q38ExpertScaleCache *expert_scales;uint64_t clock, hits, miss;+ uint64_t expert_weight_reads, expert_scale_reads, expert_pair_reads;+ uint64_t expert_prefetch_ranges, expert_parallel_batches;+ uint64_t expert_scale_bytes;float **DN_rec, **DN_conv;float **K, **V, **IK;int kv_len, kv_cap, max_t;@@ -109,7 +126,7 @@ typedef struct {float *PLE_conv_state;int ple_history_len;int range_begin, range_end;- int native_fp8, native_bf16;+ int native_fp8, native_bf16, expert_prefetch, expert_parallel_reads;uint64_t resident_weight_bytes;double dense_load_s;Q38Timers timers;@@ -167,7 +184,9 @@ static void q38_matmul(float *y, const float *x, const float *W, int S, int I, istatic void q38_weight_free(Q38Weight *weight) {if(!weight)return;- free(weight->data);free(weight->scales);memset(weight,0,sizeof(*weight));+ if(weight->owns_data)free(weight->data);+ if(weight->owns_scales)free(weight->scales);+ memset(weight,0,sizeof(*weight));}static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,int cols) {@@ -177,7 +196,8 @@ static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,intint64_t elements=(int64_t)rows*cols;int64_t scales=kind==Q38_WEIGHT_FP8?fp8_nblk(rows)*fp8_nblk(cols):0;if(weight->kind==kind&&weight->rows==rows&&weight->cols==cols&&- weight->data&&(!scales||weight->scales))return;+ weight->owns_data&&weight->data&&+ (!scales||(weight->owns_scales&&weight->scales)))return;q38_weight_free(weight);size_t element_size=kind==Q38_WEIGHT_F32?sizeof(float):kind==Q38_WEIGHT_BF16?sizeof(uint16_t):@@ -194,6 +214,7 @@ static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,int}weight->rows=rows;weight->cols=cols;weight->elements=elements;weight->scale_count=scales;weight->kind=kind;+ weight->owns_data=1;weight->owns_scales=scales?1u:0u;}static uint64_t q38_weight_bytes(const Q38Weight *weight) {@@ -623,6 +644,8 @@ static void model_init_range(Model *m,const char *snap,int cap,int bits,(void)bits; memset(m,0,sizeof(*m)); double t0=now_s();m->native_fp8=q38_env_bool("Q38_NATIVE_FP8",1);m->native_bf16=q38_env_bool("Q38_NATIVE_BF16",1);+ m->expert_prefetch=q38_env_bool("Q38_EXPERT_PREFETCH",1);+ m->expert_parallel_reads=q38_env_bool("Q38_EXPERT_PARALLEL_READS",1);q38_load_cfg(&m->c,snap); q38_validate_cfg(&m->c); st_init(&m->S,snap);Cfg *c=&m->c; char nm[320];if(st_has(&m->S,"model.language_model.embed_tokens.weight")) snprintf(m->prefix,sizeof m->prefix,"model.language_model");@@ -638,7 +661,9 @@ static void model_init_range(Model *m,const char *snap,int cap,int bits,}m->L=(Layer*)calloc((size_t)c->layers,sizeof(Layer));m->cache=(LCache*)calloc((size_t)c->layers,sizeof(LCache));- if(!m->L||!m->cache){fprintf(stderr,"OOM model metadata\n");exit(1);}+ m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers,+ sizeof(*m->expert_scales));+ if(!m->L||!m->cache||!m->expert_scales){fprintf(stderr,"OOM model metadata\n");exit(1);}for(int i=layer_begin;i<layer_end;i++){Layer *l=&m->L[i]; q38_load_gr(m,&l->attn_gr,i,"attn_hyper_connection",1); q38_load_gr(m,&l->mlp_gr,i,"mlp_hyper_connection",1);#define WLD(field,suf,o,in) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_weight(m,nm,(o),(in))@@ -719,6 +744,217 @@ static void q38_gr_apply(const Cfg *c,float *hyper,const float *block,const floa}}+typedef struct {+ st_tensor *tensor;+ int expert, projection;+} Q38ScaleDesc;++static int q38_scale_desc_cmp(const void *left,const void *right) {+ const Q38ScaleDesc *a=(const Q38ScaleDesc*)left;+ const Q38ScaleDesc *b=(const Q38ScaleDesc*)right;+ if(a->tensor->off<b->tensor->off)return -1;+ if(a->tensor->off>b->tensor->off)return 1;+ return 0;+}++static int q38_scale_group_span(Q38ScaleDesc *desc,int count,int *fd,+ int64_t *begin,int64_t *nbytes) {+ if(!desc||count<1||!fd||!begin||!nbytes)return -1;+ qsort(desc,(size_t)count,sizeof(*desc),q38_scale_desc_cmp);+ int group_fd=desc[0].tensor->fd;+ int64_t first=desc[0].tensor->off,cursor=first;+ for(int i=0;i<count;i++){+ st_tensor *tensor=desc[i].tensor;+ if(tensor->fd!=group_fd||tensor->off!=cursor||tensor->nbytes<=0||+ tensor->nbytes>INT64_MAX-cursor)return -1;+ cursor+=tensor->nbytes;+ }+ *fd=group_fd;*begin=first;*nbytes=cursor-first;return 0;+}++static void q38_decode_scale_tensor(float *out,const unsigned char *raw,+ const st_tensor *tensor) {+ if(tensor->dtype==2)memcpy(out,raw,(size_t)tensor->nbytes);+ else for(int64_t i=0;i<tensor->numel;i++){+ uint16_t half;memcpy(&half,raw+(size_t)i*sizeof(half),sizeof(half));+ out[i]=tensor->dtype==0?bf16_to_f32(half):f16_to_f32(half);+ }+}++/* The official checkpoint stores every layer's gate/up scale sidecars as one+ * compact range and every down sidecar as another. Normalize both ranges to+ * an expert-indexed F32 bank once, scattering by the numeric expert id rather+ * than the checkpoint's lexical tensor order (0, 1, 10, ...). Variants that+ * do not satisfy the compact-range invariant simply keep the established+ * per-matrix loader. */+static int q38_prepare_expert_scale_bank(Model *m,int layer) {+ Cfg *c=&m->c;+ if(!m->native_fp8||layer<0||layer>=c->layers)return 0;+ if(!m->expert_scales){+ m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers,+ sizeof(*m->expert_scales));+ if(!m->expert_scales){fprintf(stderr,"OOM expert scale metadata\n");exit(1);}+ }+ Q38ExpertScaleCache *cache=&m->expert_scales[layer];+ if(cache->ready)return cache->ready>0;+ int64_t scale_count=fp8_nblk(c->inter)*fp8_nblk(c->hidden);+ if(scale_count<1||(uint64_t)c->experts>SIZE_MAX/(3u*(uint64_t)scale_count*sizeof(float))){+ fprintf(stderr,"invalid expert scale-bank geometry\n");exit(1);+ }+ Q38ScaleDesc *gate_up=(Q38ScaleDesc*)malloc((size_t)c->experts*2*sizeof(*gate_up));+ Q38ScaleDesc *down=(Q38ScaleDesc*)malloc((size_t)c->experts*sizeof(*down));+ if(!gate_up||!down){fprintf(stderr,"OOM expert scale descriptors\n");exit(1);}+ int ngu=0,ndown=0;char suffix[192],name[320];+ const char *projection[3]={"gate_proj","up_proj","down_proj"};+ for(int expert=0;expert<c->experts;expert++)for(int kind=0;kind<3;kind++){+ int rows=kind==2?c->hidden:c->inter;+ int cols=kind==2?c->inter:c->hidden;+ int length=snprintf(suffix,sizeof suffix,+ "mlp.experts.%d.%s.weight_scale_inv",expert,projection[kind]);+ if(length<0||(size_t)length>=sizeof suffix)goto incompatible;+ q38_name(m,name,sizeof name,layer,suffix);+ st_tensor *tensor=st_find(&m->S,name);+ int64_t block_rows=fp8_nblk(rows),block_cols=fp8_nblk(cols);+ if(!tensor||tensor->dtype<0||tensor->dtype>2||tensor->rank!=2||+ tensor->shape[0]!=block_rows||tensor->shape[1]!=block_cols||+ tensor->numel!=scale_count||+ tensor->numel>INT64_MAX/st_dtype_esz(tensor->dtype)||+ tensor->nbytes!=tensor->numel*st_dtype_esz(tensor->dtype))+ goto incompatible;+ Q38ScaleDesc item={tensor,expert,kind};+ if(kind<2)gate_up[ngu++]=item;else down[ndown++]=item;+ }+ int gu_fd,down_fd;int64_t gu_begin,down_begin,gu_bytes,down_bytes;+ if(q38_scale_group_span(gate_up,ngu,&gu_fd,&gu_begin,&gu_bytes)||+ q38_scale_group_span(down,ndown,&down_fd,&down_begin,&down_bytes)||+ (uint64_t)gu_bytes>SIZE_MAX||(uint64_t)down_bytes>SIZE_MAX)+ goto incompatible;+ float *values=(float*)malloc((size_t)c->experts*3*(size_t)scale_count*sizeof(float));+ unsigned char *gu_raw=(unsigned char*)malloc((size_t)gu_bytes);+ unsigned char *down_raw=(unsigned char*)malloc((size_t)down_bytes);+ if(!values||!gu_raw||!down_raw){fprintf(stderr,"OOM resident expert scales\n");exit(1);}+ double started=now_s();+ st_read_range_raw_cap(&m->S,gu_fd,gu_begin,gu_bytes,gu_raw,gu_bytes,1,+ "pread Qwen3.8 gate/up scales");+ st_read_range_raw_cap(&m->S,down_fd,down_begin,down_bytes,down_raw,down_bytes,1,+ "pread Qwen3.8 down scales");+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_scale_reads+=2;+ for(int i=0;i<ngu;i++){+ Q38ScaleDesc *item=&gate_up[i];+ float *dst=values+((int64_t)item->expert*3+item->projection)*scale_count;+ q38_decode_scale_tensor(dst,gu_raw+(item->tensor->off-gu_begin),item->tensor);+ }+ for(int i=0;i<ndown;i++){+ Q38ScaleDesc *item=&down[i];+ float *dst=values+((int64_t)item->expert*3+item->projection)*scale_count;+ q38_decode_scale_tensor(dst,down_raw+(item->tensor->off-down_begin),item->tensor);+ }+ free(gu_raw);free(down_raw);free(gate_up);free(down);+ cache->values=values;cache->scale_count=scale_count;cache->ready=1;+ m->expert_scale_bytes+=(uint64_t)c->experts*3*(uint64_t)scale_count*sizeof(float);+ return 1;+incompatible:+ free(gate_up);free(down);cache->ready=-1;return 0;+}++static void q38_bind_borrowed_fp8(Q38Weight *weight,void *data,float *scales,+ int rows,int cols) {+ q38_weight_free(weight);+ weight->data=data;weight->scales=scales;weight->rows=rows;weight->cols=cols;+ weight->elements=(int64_t)rows*cols;+ weight->scale_count=fp8_nblk(rows)*fp8_nblk(cols);+ weight->kind=Q38_WEIGHT_FP8;+}++static void q38_bind_fp8_slot(Slot *slot,float *scales,int scale_count,+ int hidden,int intermediate) {+ int64_t matrix_bytes=(int64_t)hidden*intermediate;+ if(matrix_bytes<1||matrix_bytes>INT64_MAX/3||+ (uint64_t)(matrix_bytes*3)>SIZE_MAX||+ scale_count!=fp8_nblk(hidden)*fp8_nblk(intermediate)){+ fprintf(stderr,"invalid native FP8 expert slab geometry\n");exit(1);+ }+ q38_weight_free(&slot->gate);q38_weight_free(&slot->up);+ q38_weight_free(&slot->down);+ int64_t slab_bytes=matrix_bytes*3;+ if(!slot->fp8_slab||slot->fp8_slab_bytes!=slab_bytes){+ void *replacement=realloc(slot->fp8_slab,(size_t)slab_bytes);+ if(!replacement){fprintf(stderr,"OOM native FP8 expert slab\n");exit(1);}+ slot->fp8_slab=replacement;slot->fp8_slab_bytes=slab_bytes;+ }+ unsigned char *raw=(unsigned char*)slot->fp8_slab;+ q38_bind_borrowed_fp8(&slot->gate,raw,scales,intermediate,hidden);+ q38_bind_borrowed_fp8(&slot->up,raw+matrix_bytes,scales+scale_count,+ intermediate,hidden);+ q38_bind_borrowed_fp8(&slot->down,raw+2*matrix_bytes,scales+2*scale_count,+ hidden,intermediate);+}++static int q38_native_fp8_expert_tensors(Model *m,int layer,int expert,+ st_tensor *weight[3]) {+ if(!m->native_fp8)return 0;+ Cfg *c=&m->c;const char *projection[3]={"gate_proj","up_proj","down_proj"};+ int rows[3]={c->inter,c->inter,c->hidden};+ int cols[3]={c->hidden,c->hidden,c->inter};+ char suffix[192],name[320];+ for(int kind=0;kind<3;kind++){+ int length=snprintf(suffix,sizeof suffix,"mlp.experts.%d.%s.weight",+ expert,projection[kind]);+ if(length<0||(size_t)length>=sizeof suffix)return 0;+ q38_name(m,name,sizeof name,layer,suffix);weight[kind]=st_find(&m->S,name);+ if(!weight[kind]||weight[kind]->dtype!=4||weight[kind]->rank!=2||+ weight[kind]->shape[0]!=rows[kind]||weight[kind]->shape[1]!=cols[kind]||+ weight[kind]->nbytes!=(int64_t)rows[kind]*cols[kind])return 0;+ }+ if(weight[0]->fd!=weight[1]->fd||+ weight[0]->off>INT64_MAX-weight[0]->nbytes||+ weight[1]->off!=weight[0]->off+weight[0]->nbytes)return 0;+ return 1;+}++static void q38_load_native_fp8_ranges(Model *m,int layer,int expert,Slot *slot,+ st_tensor *weight[3]) {+ Cfg *c=&m->c;+ Q38ExpertScaleCache *cache=&m->expert_scales[layer];+ float *scales=cache->values+(int64_t)expert*3*cache->scale_count;+ q38_bind_fp8_slot(slot,scales,(int)cache->scale_count,c->hidden,c->inter);+ int64_t pair_bytes=weight[0]->nbytes+weight[1]->nbytes;+ unsigned char *raw=(unsigned char*)slot->fp8_slab;+ st_read_range_raw_cap(&m->S,weight[0]->fd,weight[0]->off,pair_bytes,+ raw,pair_bytes,1,"pread Qwen3.8 gate/up expert");+ st_read_range_raw_cap(&m->S,weight[2]->fd,weight[2]->off,weight[2]->nbytes,+ raw+pair_bytes,slot->fp8_slab_bytes-pair_bytes,1,+ "pread Qwen3.8 down expert");+}++static int q38_try_load_native_fp8_expert(Model *m,int layer,int expert,Slot *slot) {+ st_tensor *weight[3];+ if(!q38_native_fp8_expert_tensors(m,layer,expert,weight)||+ !q38_prepare_expert_scale_bank(m,layer))return 0;+ double started=now_s();+ q38_load_native_fp8_ranges(m,layer,expert,slot,weight);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ m->expert_weight_reads+=2;m->expert_pair_reads++;+ return 1;+}++static void q38_prefetch_native_fp8_experts(Model *m,int layer,+ const int *experts,int count) {+ if(!m->expert_prefetch||!m->native_fp8||!experts||count<1||+ !q38_prepare_expert_scale_bank(m,layer))return;+ LCache *cache=&m->cache[layer];+ for(int index=0;index<count;index++){+ int expert=experts[index];st_tensor *weight[3];+ if(expert<0||expert>=m->c.experts||cache->by_expert[expert]>=0||+ !q38_native_fp8_expert_tensors(m,layer,expert,weight))continue;+ posix_fadvise(weight[0]->fd,weight[0]->off,+ weight[0]->nbytes+weight[1]->nbytes,POSIX_FADV_WILLNEED);+ posix_fadvise(weight[2]->fd,weight[2]->off,weight[2]->nbytes,+ POSIX_FADV_WILLNEED);+ m->expert_prefetch_ranges+=2;+ }+}+static void q38_load_fp8_expert_weight(Model *m,const char *wn,const char *sn,Q38Weight *out,int O,int I) {st_tensor *w=st_find(&m->S,wn),*sc=st_find(&m->S,sn);@@ -735,6 +971,7 @@ static void q38_load_fp8_expert_weight(Model *m,const char *wn,const char *sn,st_read_raw_cap(&m->S,wn,out->data,w->nbytes,1);st_read_f32(&m->S,sn,out->scales,1);q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ m->expert_weight_reads++;m->expert_scale_reads++;return;}q38_weight_reserve(out,Q38_WEIGHT_F32,O,I);@@ -742,7 +979,8 @@ static void q38_load_fp8_expert_weight(Model *m,const char *wn,const char *sn,if(!raw){fprintf(stderr,"OOM FP8 expert staging\n");exit(1);}double started=now_s();st_read_raw_cap(&m->S,wn,raw,w->nbytes,1);st_read_f32(&m->S,sn,scale,1);- q38_tm_add(m,Q38_TM_EXPERT_READ,started); started=now_s();+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ m->expert_weight_reads++;m->expert_scale_reads++;started=now_s();float *decoded=(float*)out->data;#pragma omp parallel for schedule(static)for(int o=0;o<O;o++)for(int i=0;i<I;i++)@@ -764,7 +1002,7 @@ static void q38_load_expert_weight(Model *m,const char *name,Q38Weight *out,if(kind==Q38_WEIGHT_BF16)st_read_raw_cap(&m->S,name,out->data,tensor->nbytes,1);else st_read_f32(&m->S,name,(float*)out->data,1);- q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;}static void q38_load_expert_slice(Model *m,const char *name,const st_tensor *tensor,@@ -780,7 +1018,7 @@ static void q38_load_expert_slice(Model *m,const char *name,const st_tensor *tenst_read_slice_raw_cap(&m->S,name,element_offset*(int64_t)sizeof(uint16_t),bytes,out->data,bytes,1);} else st_read_slice_f32(&m->S,name,element_offset,(int64_t)O*I,(float*)out->data,1);- q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;}static void q38_load_expert(Model *m,int layer,int eid,Slot *s) {@@ -797,6 +1035,7 @@ static void q38_load_expert(Model *m,int layer,int eid,Slot *s) {q38_load_expert_slice(m,nm,t,(int64_t)eid*H*I,&s->down,H,I);return;}+ if(q38_try_load_native_fp8_expert(m,layer,eid,s))return;const char *kind[3]={"gate_proj","up_proj","down_proj"};Q38Weight *dst[3]={&s->gate,&s->up,&s->down};int os[3]={I,I,H},is[3]={H,H,I};for(int k=0;k<3;k++){@@ -819,6 +1058,97 @@ static Slot *q38_expert_get(Model *m,int layer,int eid) {s->eid=-1;q38_load_expert(m,layer,eid,s);s->eid=eid;s->used=++m->clock;lc->by_expert[eid]=(int)(s-lc->slots);return s;}+typedef struct {+ int expert;+ Slot *slot;+ st_tensor *weight[3];+} Q38ExpertLoadJob;++/* Reserve an entire routed demand set on the main thread, fill distinct slots+ * in parallel, then publish every cache index on the main thread. Demand-set+ * residents are protected from victim selection, so no worker can overwrite a+ * slot another selected expert will consume. Smaller caches and heterogeneous+ * layouts retain the serial LRU path. */+static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,+ Slot **selected) {+ if(!m->expert_parallel_reads||!experts||!selected||count<2||+ count>Q38_MAX_TOPK)return 0;+ LCache *cache=&m->cache[layer];+ if(cache->cap<count||!q38_prepare_expert_scale_bank(m,layer))return 0;+ Q38ExpertLoadJob jobs[Q38_MAX_TOPK];+ for(int index=0;index<count;index++){+ int expert=experts[index];+ if(expert<0||expert>=m->c.experts)return 0;+ for(int previous=0;previous<index;previous++)+ if(experts[previous]==expert)return 0;+ int slot_index=cache->by_expert[expert];+ if(slot_index>=0){+ if(slot_index>=cache->n||cache->slots[slot_index].eid!=expert)return 0;+ continue;+ }+ st_tensor *weight[3];+ if(!q38_native_fp8_expert_tensors(m,layer,expert,weight))return 0;+ }+ unsigned char *protected_slots=(unsigned char*)calloc((size_t)cache->cap,1);+ if(!protected_slots){fprintf(stderr,"OOM expert batch reservations\n");exit(1);}+ for(int index=0;index<count;index++){+ int slot_index=cache->by_expert[experts[index]];+ if(slot_index>=0)protected_slots[slot_index]=1;+ }+ int job_count=0;+ for(int index=0;index<count;index++){+ int expert=experts[index],slot_index=cache->by_expert[expert];Slot *slot;+ if(slot_index>=0){+ slot=&cache->slots[slot_index];m->hits++;+ }else{+ m->miss++;+ if(cache->n<cache->cap){+ slot=&cache->slots[cache->n++];slot->eid=-1;+ }else{+ int victim=-1;+ for(int candidate=0;candidate<cache->n;candidate++)+ if(!protected_slots[candidate]&&+ (victim<0||cache->slots[candidate].used<cache->slots[victim].used))+ victim=candidate;+ if(victim<0){+ fprintf(stderr,"Qwen3.8 expert demand set has no reservable cache slot\n");+ exit(1);+ }+ slot=&cache->slots[victim];+ if(slot->eid>=0)cache->by_expert[slot->eid]=-1;+ }+ slot->eid=-1;jobs[job_count].expert=expert;jobs[job_count].slot=slot;+ if(!q38_native_fp8_expert_tensors(m,layer,expert,jobs[job_count].weight)){+ fprintf(stderr,"Qwen3.8 expert layout changed during batch reservation\n");exit(1);+ }+ job_count++;+ }+ slot->used=++m->clock;selected[index]=slot;+ protected_slots[slot-cache->slots]=1;+ }+ free(protected_slots);+ if(job_count){+ int workers=job_count;+#ifdef _OPENMP+ int thread_limit=omp_get_max_threads();if(workers>thread_limit)workers=thread_limit;+#endif+ double started=now_s();+ #pragma omp parallel for schedule(static) num_threads(workers) if(job_count>1)+ for(int job=0;job<job_count;job++)+ q38_load_native_fp8_ranges(m,layer,jobs[job].expert,jobs[job].slot,+ jobs[job].weight);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ m->expert_weight_reads+=(uint64_t)job_count*2;+ m->expert_pair_reads+=(uint64_t)job_count;+ if(job_count>1)m->expert_parallel_batches++;+ for(int job=0;job<job_count;job++){+ Slot *slot=jobs[job].slot;slot->eid=jobs[job].expert;+ cache->by_expert[jobs[job].expert]=(int)(slot-cache->slots);+ }+ }+ return 1;+}+static void q38_ple_row(Model *m,int64_t row,float *out) {Cfg *c=&m->c; int p=0;while(p+1<m->ple_part_count&&row>=m->ple_part_start[p+1])p++;@@ -990,19 +1320,22 @@ static void q38_moe(Model *m,Layer *l,int layer,const float *x,int S,float *out)float route_gates[Q38_MAX_TOPK];for(int z=0;z<K;z++) route_gates[z]=(float)(val[z]/den);rt_route(layer,s,idx,route_gates,K); /* shared counts + post-normalization trace */+ q38_prefetch_native_fp8_experts(m,layer,idx,K);+ double phase_started=now_s();+ q38_weight_matmul(sg,xs,&l->sh_g,1,H,SI);q38_weight_matmul(su,xs,&l->sh_u,1,H,SI);+ for(int j=0;j<SI;j++)sh[j]=q38_silu(sg[j])*su[j];q38_weight_matmul(shared,sh,&l->sh_d,1,SI,H);+ float gate=0.f;for(int d=0;d<H;d++)gate+=xs[d]*l->sh_gate[d];gate=q38_sigmoid(gate);+ q38_tm_add(m,Q38_TM_SHARED_EXPERT,phase_started);+ Slot *selected[Q38_MAX_TOPK];+ int loaded_batch=q38_expert_get_batch(m,layer,idx,K,selected);for(int z=0;z<K;z++){- Slot *ex=q38_expert_get(m,layer,idx[z]); double phase_started=now_s();+ Slot *ex=loaded_batch?selected[z]:q38_expert_get(m,layer,idx[z]);phase_started=now_s();q38_weight_matmul(eg,xs,&ex->gate,1,H,I);q38_weight_matmul(eu,xs,&ex->up,1,H,I);for(int j=0;j<I;j++)eh[j]=q38_silu(eg[j])*eu[j];q38_weight_matmul(eo,eh,&ex->down,1,I,H);for(int d=0;d<H;d++)ys[d]+=route_gates[z]*eo[d];q38_tm_add(m,Q38_TM_ROUTED_EXPERT,phase_started);}- double phase_started=now_s();- q38_weight_matmul(sg,xs,&l->sh_g,1,H,SI);q38_weight_matmul(su,xs,&l->sh_u,1,H,SI);- for(int j=0;j<SI;j++)sh[j]=q38_silu(sg[j])*su[j];q38_weight_matmul(shared,sh,&l->sh_d,1,SI,H);- float gate=0.f;for(int d=0;d<H;d++)gate+=xs[d]*l->sh_gate[d];gate=q38_sigmoid(gate);for(int d=0;d<H;d++)ys[d]+=gate*shared[d];- q38_tm_add(m,Q38_TM_SHARED_EXPERT,phase_started);}rt_trace_end();free(logits);free(sg);free(su);free(sh);free(shared);free(eg);free(eu);free(eh);free(eo);@@ -1101,6 +1434,16 @@ static void q38_tm_report_bank(const Q38Timers *timers,const char *scope) {static void tm_report(const Model *m) {q38_tm_report_bank(&m->timers,"total");+ if(q38_tm_enabled())+ fprintf(stderr,"[qwen38 expert I/O] weight-ranges=%llu scale-ranges=%llu "+ "coalesced-gate-up=%llu prefetched=%llu parallel-batches=%llu "+ "resident-scales=%.2f MiB\n",+ (unsigned long long)m->expert_weight_reads,+ (unsigned long long)m->expert_scale_reads,+ (unsigned long long)m->expert_pair_reads,+ (unsigned long long)m->expert_prefetch_ranges,+ (unsigned long long)m->expert_parallel_batches,+ m->expert_scale_bytes/1048576.0);}static void q38_layer_free(Layer *l) {@@ -1126,13 +1469,15 @@ static void q38_model_free(Model *m) {q38_weight_free(&m->cache[i].slots[s].gate);q38_weight_free(&m->cache[i].slots[s].up);q38_weight_free(&m->cache[i].slots[s].down);+ free(m->cache[i].slots[s].fp8_slab);}free(m->cache[i].slots); free(m->cache[i].by_expert);}+ if(m->expert_scales)free(m->expert_scales[i].values);free(m->DN_rec ? m->DN_rec[i] : NULL); free(m->DN_conv ? m->DN_conv[i] : NULL);free(m->K ? m->K[i] : NULL); free(m->V ? m->V[i] : NULL); free(m->IK ? m->IK[i] : NULL);}- free(m->L); free(m->cache); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);+ free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);q38_weight_free(&m->embed);q38_weight_free(&m->lm_head);free(m->final_gr.norm);q38_weight_free(&m->final_gr.down);q38_weight_free(&m->final_gr.up);q38_weight_free(&m->final_gr.inject);free(m->ple_history); free(m->PLE_conv_state); free(m->c.is_attn); st_destroy(&m->S);diff --git a/c/st.h b/c/st.hindex bfdea716..9f6a55f3 100644--- a/c/st.h+++ b/c/st.h@@ -938,6 +938,30 @@ static void st_read_raw_cap(shards *S, const char *name, void *out, int64_t cap,st_read_raw(S, name, out, drop);}+/* Read an exact physical range from one indexed shard. This is intentionally+ * separate from tensor slices: callers may use it only after validating a+ * checkpoint-specific packing invariant (for example, two adjacent tensors).+ * The shard membership, file bounds and destination capacity remain enforced+ * here so a malformed header cannot turn that optimization into an unchecked+ * pread. */+static void st_read_range_raw_cap(shards *S, int fd, int64_t off,+ int64_t nbytes, void *out, int64_t cap,+ int drop, const char *tag) {+ int fidx=S?st_fidx(S,fd):-1;+ if(fidx<0){fprintf(stderr,"physical range uses an unindexed shard fd\n");exit(1);}+ if(off<0||nbytes<0||cap<0||nbytes>cap||+ off>S->sizes[fidx]||nbytes>S->sizes[fidx]-off||+ (uint64_t)nbytes>SIZE_MAX){+ fprintf(stderr,"physical shard range [%lld,+%lld) exceeds file/destination bounds "+ "(file %lld, cap %lld) — refusing (untrusted container)\n",+ (long long)off,(long long)nbytes,(long long)S->sizes[fidx],+ (long long)cap);exit(1);+ }+ if(nbytes>0&&!out){fprintf(stderr,"physical range has a NULL destination\n");exit(1);}+ if(nbytes>0)st_pread_full(fd,out,nbytes,off,tag&&*tag?tag:"pread physical range");+ if(drop&&nbytes>0)posix_fadvise(fd,off,nbytes,POSIX_FADV_DONTNEED);+}+/* Read-only view of one tensor's exact stored bytes. Unlike st_read_raw this* performs no allocation or copy; unlike a naked mmap pointer it carries the* aligned OS view/handle required for cleanup. Callers must still validatediff --git a/c/tests/test_qwen38_native_weights.c b/c/tests/test_qwen38_native_weights.cindex eaadb18f..9889ded6 100644--- a/c/tests/test_qwen38_native_weights.c+++ b/c/tests/test_qwen38_native_weights.c@@ -20,7 +20,7 @@ typedef struct {unsigned char raw[24];size_t raw_bytes;float scale;- int is_scale;+ int expert,projection,is_scale;} FixtureTensor;static uint16_t to_bf16(float value){@@ -61,10 +61,12 @@ static int add_fixture_tensor(FixtureTensor *tensors,int *count,int64_t *offset,int length=snprintf(tensor->name,sizeof tensor->name,"%s",name);if(length<0||(size_t)length>=sizeof tensor->name)return -1;tensor->dtype=dtype;tensor->rows=rows;tensor->cols=cols;- tensor->begin=*offset;tensor->is_scale=is_scale;+ tensor->begin=*offset;tensor->expert=expert;tensor->projection=projection;+ tensor->is_scale=is_scale;if(is_scale){tensor->scale=0.125f*(float)(1+expert*3+projection);- *offset+=(int64_t)sizeof(float);+ *offset+=!strcmp(dtype,"F32")?(int64_t)sizeof(float):+ (int64_t)sizeof(uint16_t);}else if(!strcmp(dtype,"BF16")){tensor->raw_bytes=(size_t)rows*cols*sizeof(uint16_t);for(int index=0;index<rows*cols;index++){@@ -81,7 +83,7 @@ static int add_fixture_tensor(FixtureTensor *tensors,int *count,int64_t *offset,tensor->end=*offset;return 0;}-static int write_fp8_fixture(const char *directory,int mixed){+static int write_fp8_fixture(const char *directory,int mixed,int adjacent){FixtureTensor tensors[12];int count=0;int64_t offset=0;const char *projection[3]={"gate_proj","up_proj","down_proj"};for(int expert=0;expert<2;expert++)for(int kind=0;kind<3;kind++){@@ -97,10 +99,41 @@ static int write_fp8_fixture(const char *directory,int mixed){"model.layers.0.mlp.experts.%d.%s.weight_scale_inv",expert,projection[kind]);if(length<0||(size_t)length>=sizeof name||- add_fixture_tensor(tensors,&count,&offset,name,"F32",1,1,+ add_fixture_tensor(tensors,&count,&offset,name,adjacent?"BF16":"F32",1,1,expert,kind,1))return -1;}+ int order[12];for(int index=0;index<count;index++)order[index]=index;+ if(adjacent){+ /* Match the official packing invariant while deliberately keeping the+ * JSON entries in construction order: compact gate/up sidecars, compact+ * down sidecars, then gate+up payload pairs. The production loader+ * must use offsets and numeric expert ids, not header order. */+ for(int left=1;left<count;left++){+ int value=order[left],right=left-1;+ FixtureTensor *v=&tensors[value];+ int vcat=v->is_scale?(v->projection<2?0:1):2;+ int vkey=vcat*100+v->expert*3+v->projection;+ while(right>=0){+ FixtureTensor *r=&tensors[order[right]];+ int rcat=r->is_scale?(r->projection<2?0:1):2;+ int rkey=rcat*100+r->expert*3+r->projection;+ if(rkey<=vkey)break;+ order[right+1]=order[right];right--;+ }+ order[right+1]=value;+ }+ offset=0;+ for(int physical=0;physical<count;physical++){+ FixtureTensor *tensor=&tensors[order[physical]];+ int64_t bytes=tensor->is_scale?+ (!strcmp(tensor->dtype,"F32")?(int64_t)sizeof(float):+ (int64_t)sizeof(uint16_t)):+ (int64_t)tensor->raw_bytes;+ tensor->begin=offset;tensor->end=offset+bytes;offset+=bytes;+ }+ }+char header[8192];size_t used=0;header[used++]='{';for(int index=0;index<count;index++){FixtureTensor *tensor=&tensors[index];@@ -121,10 +154,14 @@ static int write_fp8_fixture(const char *directory,int mixed){uint64_t header_length=used;int failed=fwrite(&header_length,sizeof header_length,1,file)!=1||fwrite(header,1,used,file)!=used;- for(int index=0;index<count&&!failed;index++){- FixtureTensor *tensor=&tensors[index];- if(tensor->is_scale)+ for(int physical=0;physical<count&&!failed;physical++){+ FixtureTensor *tensor=&tensors[order[physical]];+ if(tensor->is_scale&&!strcmp(tensor->dtype,"F32"))failed=fwrite(&tensor->scale,sizeof tensor->scale,1,file)!=1;+ else if(tensor->is_scale){+ uint16_t scale=to_bf16(tensor->scale);+ failed=fwrite(&scale,sizeof scale,1,file)!=1;+ }elsefailed=fwrite(tensor->raw,1,tensor->raw_bytes,file)!=tensor->raw_bytes;}@@ -155,10 +192,13 @@ static void destroy_fixture_model(Model *model){q38_weight_free(&model->cache[0].slots[slot].gate);q38_weight_free(&model->cache[0].slots[slot].up);q38_weight_free(&model->cache[0].slots[slot].down);+ free(model->cache[0].slots[slot].fp8_slab);}free(model->cache[0].slots);free(model->cache[0].by_expert);}- free(model->cache);st_destroy(&model->S);memset(model,0,sizeof(*model));+ if(model->expert_scales)free(model->expert_scales[0].values);+ free(model->expert_scales);free(model->cache);st_destroy(&model->S);+ memset(model,0,sizeof(*model));}static uint8_t fixture_fp8_byte(int expert,int projection,int index){@@ -204,25 +244,30 @@ static int verify_loaded_fp8_slot(const Slot *slot,int expert,int native_fp8){verify_loaded_fp8_weight(&slot->down,expert,2,native_fp8)?-1:0;}-static int check_fixture_mode(const char *directory,int native_fp8){+static int check_fixture_mode(const char *directory,int native_fp8,int expect_fast){Model model;if(init_fixture_model(&model,directory,native_fp8))return -1;int result=-1;- uint64_t bytes_per_capacity=0;unsigned numeric_kinds=0;- if(q38_segment_expert_layout(&model,0,1,&bytes_per_capacity,&numeric_kinds))+ uint64_t bytes_per_capacity=0,fixed_scale_bytes=0;unsigned numeric_kinds=0;+ if(q38_segment_expert_layout(&model,0,1,&bytes_per_capacity,+ &fixed_scale_bytes,&numeric_kinds))goto cleanup;char numeric_class[96];q38_segment_numeric_class(numeric_class,sizeof numeric_class,numeric_kinds);if(native_fp8){- if(bytes_per_capacity!=30||numeric_kinds!=Q38_EXPERT_FP8_BLOCK||+ if(bytes_per_capacity!=18||fixed_scale_bytes!=24||+ numeric_kinds!=Q38_EXPERT_FP8_BLOCK||strcmp(numeric_class,"qwen38/fp8-block-f32dot-f64blocksum/cpu-v1"))goto cleanup;- }else if(bytes_per_capacity!=72||numeric_kinds!=Q38_EXPERT_FP8_EXPANDED||+ }else if(bytes_per_capacity!=72||fixed_scale_bytes||+ numeric_kinds!=Q38_EXPERT_FP8_EXPANDED||strcmp(numeric_class,"qwen38/fp8-expanded-f32dot/cpu-v1"))goto cleanup;int capacity=0;- if(q38_segment_cache_capacity(bytes_per_capacity,2,bytes_per_capacity*2,+ if(q38_segment_cache_capacity(bytes_per_capacity,fixed_scale_bytes,2,+ fixed_scale_bytes+bytes_per_capacity*2,&capacity)||capacity!=2||- !q38_segment_cache_capacity(bytes_per_capacity,2,bytes_per_capacity-1,+ !q38_segment_cache_capacity(bytes_per_capacity,fixed_scale_bytes,2,+ fixed_scale_bytes+bytes_per_capacity-1,&capacity))goto cleanup;if(q38_segment_cache_resize(&model,2)||model.cache[0].cap!=2||q38_segment_cache_resize(&model,1)||model.cache[0].cap!=1)goto cleanup;@@ -232,6 +277,15 @@ static int check_fixture_mode(const char *directory,int native_fp8){if(first->gate.kind!=expected||first->up.kind!=expected||first->down.kind!=expected||model.miss!=1||model.hits!=0||verify_loaded_fp8_slot(first,0,native_fp8))goto cleanup;+ void *slab=first->fp8_slab;+ if(expect_fast){+ if(!native_fp8||!slab||first->gate.data!=slab||+ first->up.data!=(unsigned char*)slab+6||+ first->down.data!=(unsigned char*)slab+12||+ model.expert_weight_reads!=2||model.expert_scale_reads!=2||+ model.expert_pair_reads!=1||model.expert_scale_bytes!=24)+ goto cleanup;+ }else if(model.expert_pair_reads||slab)goto cleanup;float first_value=native_fp8?e4m3_decode(((uint8_t*)first->gate.data)[0])*first->gate.scales[0]:((float*)first->gate.data)[0];@@ -240,12 +294,18 @@ static int check_fixture_mode(const char *directory,int native_fp8){if(second!=first||model.cache[0].by_expert[0]!=-1||model.cache[0].by_expert[1]!=0||model.miss!=2||model.hits!=1||verify_loaded_fp8_slot(second,1,native_fp8))goto cleanup;+ if(expect_fast&&(second->fp8_slab!=slab||model.expert_weight_reads!=4||+ model.expert_scale_reads!=2||model.expert_pair_reads!=2))+ goto cleanup;float second_value=native_fp8?e4m3_decode(((uint8_t*)second->gate.data)[0])*second->gate.scales[0]:((float*)second->gate.data)[0];if(first_value==second_value)goto cleanup;if(q38_expert_get(&model,0,0)!=first||model.cache[0].by_expert[0]!=0||model.cache[0].by_expert[1]!=-1||model.miss!=3)goto cleanup;+ if(expect_fast&&(first->fp8_slab!=slab||model.expert_weight_reads!=6||+ model.expert_scale_reads!=2||model.expert_pair_reads!=3))+ goto cleanup;result=0;cleanup:destroy_fixture_model(&model);return result;@@ -255,10 +315,10 @@ static int check_mixed_layout(const char *directory,int native_fp8,uint64_t expected_bytes,unsigned expected_kinds,const char *expected_class){Model model;if(init_fixture_model(&model,directory,native_fp8))return -1;- uint64_t bytes=0;unsigned kinds=0;char numeric_class[96];int result=-1;- if(!q38_segment_expert_layout(&model,0,1,&bytes,&kinds)){+ uint64_t bytes=0,fixed=0;unsigned kinds=0;char numeric_class[96];int result=-1;+ if(!q38_segment_expert_layout(&model,0,1,&bytes,&fixed,&kinds)){q38_segment_numeric_class(numeric_class,sizeof numeric_class,kinds);- if(bytes==expected_bytes&&kinds==expected_kinds&&+ if(bytes==expected_bytes&&!fixed&&kinds==expected_kinds&&!strcmp(numeric_class,expected_class)){Slot *slot=q38_expert_get(&model,0,0);if(slot&&!verify_loaded_fp8_slot(slot,0,native_fp8)&&@@ -284,6 +344,29 @@ static int check_mixed_layout(const char *directory,int native_fp8,destroy_fixture_model(&model);return result;}+static int check_parallel_batch(const char *directory){+ Model model;if(init_fixture_model(&model,directory,1))return -1;+ int result=-1;model.expert_parallel_reads=1;+ int first_ids[2]={1,0};Slot *first[2]={0};+ if(q38_expert_get_batch(&model,0,first_ids,2,first)||model.cache[0].n||+ model.miss||model.hits)goto cleanup;+ if(q38_segment_cache_resize(&model,2))goto cleanup;+ if(!q38_expert_get_batch(&model,0,first_ids,2,first)||!first[0]||!first[1]||+ first[0]==first[1]||first[0]->eid!=1||first[1]->eid!=0||+ model.miss!=2||model.hits||model.expert_parallel_batches!=1||+ model.expert_weight_reads!=4||model.expert_scale_reads!=2||+ model.expert_pair_reads!=2||verify_loaded_fp8_slot(first[0],1,1)||+ verify_loaded_fp8_slot(first[1],0,1))goto cleanup;+ int second_ids[2]={0,1};Slot *second[2]={0};+ if(!q38_expert_get_batch(&model,0,second_ids,2,second)||+ second[0]!=first[1]||second[1]!=first[0]||model.miss!=2||+ model.hits!=2||model.expert_parallel_batches!=1||+ model.expert_weight_reads!=4)goto cleanup;+ result=0;+cleanup:+ destroy_fixture_model(&model);return result;+}+static int check_segment_failure_outputs(void){char error[128];void *output=(void*)(uintptr_t)1;ColiSegmentCapabilities capabilities;@@ -339,9 +422,9 @@ int main(void){q38_weight_free(&bf16);char directory[]="test_qwen38_native_XXXXXX";CHECK(mkdtemp(directory)!=NULL);- CHECK(write_fp8_fixture(directory,0)==0);- CHECK(check_fixture_mode(directory,1)==0);- CHECK(check_fixture_mode(directory,0)==0);+ CHECK(write_fp8_fixture(directory,0,0)==0);+ CHECK(check_fixture_mode(directory,1,0)==0);+ CHECK(check_fixture_mode(directory,0,0)==0);char path[512];int path_length=snprintf(path,sizeof path,"%s/model.safetensors",directory);CHECK(path_length>0&&(size_t)path_length<sizeof path);@@ -349,7 +432,7 @@ int main(void){char mixed_directory[]="test_qwen38_mixed_XXXXXX";CHECK(mkdtemp(mixed_directory)!=NULL);- CHECK(write_fp8_fixture(mixed_directory,1)==0);+ CHECK(write_fp8_fixture(mixed_directory,1,0)==0);CHECK(check_mixed_layout(mixed_directory,1,32,Q38_EXPERT_FP8_BLOCK|Q38_EXPERT_BF16,"qwen38/mixed-expert-arithmetic-05/cpu-v1")==0);@@ -360,8 +443,18 @@ int main(void){CHECK(path_length>0&&(size_t)path_length<sizeof path);CHECK(remove(path)==0);CHECK(rmdir(mixed_directory)==0);+ char adjacent_directory[]="test_qwen38_adjacent_XXXXXX";+ CHECK(mkdtemp(adjacent_directory)!=NULL);+ CHECK(write_fp8_fixture(adjacent_directory,0,1)==0);+ CHECK(check_fixture_mode(adjacent_directory,1,1)==0);+ CHECK(check_fixture_mode(adjacent_directory,0,0)==0);+ CHECK(check_parallel_batch(adjacent_directory)==0);+ path_length=snprintf(path,sizeof path,"%s/model.safetensors",adjacent_directory);+ CHECK(path_length>0&&(size_t)path_length<sizeof path);+ CHECK(remove(path)==0);CHECK(rmdir(adjacent_directory)==0);+CHECK(check_segment_failure_outputs()==0);- puts("qwen38 native weights: independent FP8 oracle, real loader, LRU, mixed classes, adapter failures: ok");+ puts("qwen38 native weights: FP8 oracle, coalesced slabs/scales, parallel demand sets, fallbacks, adapter failures: ok");return 0;}diff --git a/c/tests/test_qwen38_tokenizer.c b/c/tests/test_qwen38_tokenizer.cindex 393e50f1..c851cf41 100644--- a/c/tests/test_qwen38_tokenizer.c+++ b/c/tests/test_qwen38_tokenizer.c@@ -160,9 +160,9 @@ int main(void) {uint8_t layer_kinds[2]={0,1};Cfg cfg={0};cfg.layers=2;cfg.is_attn=layer_kinds;cfg.hidden=10;cfg.inter=2;cfg.experts=9;int cache_capacity=0;- CHECK(q38_segment_cache_capacity(480,cfg.experts,960,&cache_capacity)==0);+ CHECK(q38_segment_cache_capacity(480,0,cfg.experts,960,&cache_capacity)==0);CHECK(cache_capacity==2);- CHECK(q38_segment_cache_capacity(480,cfg.experts,479,&cache_capacity)<0);+ CHECK(q38_segment_cache_capacity(480,0,cfg.experts,479,&cache_capacity)<0);cfg.dn_vheads=2;cfg.dn_kdim=3;cfg.dn_vdim=4;cfg.dn_conv_dim=5;cfg.dn_convk=3;cfg.kv_heads=2;cfg.head_dim=4;cfg.idx_dim=3;cfg.ple_layer=0;cfg.hc_width=8;cfg.ple_convk=3;cfg.ngram_size=3;