Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(qwen38): overlap native FP8 expert loads

themorlock99deb09011cd
diff --git a/c/qwen38.c b/c/qwen38.c
index 7cf950d2..244c4ef4 100644
--- a/c/qwen38.c
+++ b/c/qwen38.c
@@ -10,7 +10,8 @@
* block-FP8 experts remain native in a bounded per-layer LRU; the 51B-parameter
* PLE table remains on disk and only its sixteen 160-byte rows/token are read.
*
- * Runtime: SNAP=<dir>, optional Q38_MAXT, and argv
+ * Runtime: SNAP=<dir>, optional Q38_MAXT, Q38_EXPERT_PREFETCH=0 and
+ * Q38_EXPERT_PARALLEL_READS=0 A/B controls, and argv
* `qwen38 <cache/layer> <reserved> [ref.json]`. The second positional remains
* reserved so Colibri's common cache-aware launcher ABI stays stable.
*/
@@ -1625,10 +1626,11 @@ static int q38_segment_fused_matrix_bytes(Model *m,const st_tensor *tensor,
* representative. Fused containers have one dtype/geometry for every expert. */
static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,
uint64_t *bytes_per_capacity,
+ uint64_t *fixed_scale_bytes,
unsigned *numeric_kinds){
- if(!m||!bytes_per_capacity||!numeric_kinds||begin>=end||
+ if(!m||!bytes_per_capacity||!fixed_scale_bytes||!numeric_kinds||begin>=end||
end>(uint32_t)m->c.layers)return -1;
- Cfg *c=&m->c;uint64_t range_bytes=0;unsigned kinds=0;
+ Cfg *c=&m->c;uint64_t range_bytes=0,range_scales=0;unsigned kinds=0;
for(uint32_t layer=begin;layer<end;layer++){
char name[320];q38_name(m,name,sizeof name,(int)layer,
"mlp.experts.gate_up_proj");
@@ -1652,6 +1654,7 @@ static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,
const char *projection[3]={"gate_proj","up_proj","down_proj"};
int rows[3]={c->inter,c->inter,c->hidden};
int cols[3]={c->hidden,c->hidden,c->inter};
+ int all_native_fp8=m->native_fp8;
for(int expert=0;expert<c->experts;expert++){
uint64_t expert_bytes=0;
for(int projection_index=0;projection_index<3;projection_index++){
@@ -1661,6 +1664,8 @@ static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,
projection[projection_index]);
if(length<0||(size_t)length>=sizeof suffix)return -1;
q38_name(m,name,sizeof name,(int)layer,suffix);
+ st_tensor *weight=st_find(&m->S,name);
+ if(!weight||weight->dtype!=4)all_native_fp8=0;
uint64_t part;
if(q38_segment_matrix_bytes(m,name,rows[projection_index],
cols[projection_index],&part,&kinds)||
@@ -1668,17 +1673,37 @@ static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,
}
if(expert_bytes>layer_bytes)layer_bytes=expert_bytes;
}
+ if(all_native_fp8){
+ uint64_t scale_count=(uint64_t)fp8_nblk(c->inter)*
+ (uint64_t)fp8_nblk(c->hidden);
+ uint64_t per_slot_scales,layer_scale_bank;
+ if(q38_u64_mul(3u,scale_count,&per_slot_scales)||
+ q38_u64_mul(per_slot_scales,sizeof(float),&per_slot_scales)||
+ layer_bytes<per_slot_scales||
+ q38_u64_mul(per_slot_scales,(uint64_t)c->experts,
+ &layer_scale_bank)||
+ q38_u64_add(&range_scales,layer_scale_bank))return -1;
+ /* Eligible checkpoints borrow normalized scales from one
+ * per-layer bank, so only raw FP8 payload belongs to a slot.
+ * Charging the full bank even for a repacked checkpoint that
+ * later falls back is conservative and keeps tight Segment
+ * memory limits safe. */
+ layer_bytes-=per_slot_scales;
+ }
}
if(!layer_bytes||q38_u64_add(&range_bytes,layer_bytes))return -1;
}
- *bytes_per_capacity=range_bytes;*numeric_kinds=kinds;return 0;
+ *bytes_per_capacity=range_bytes;*fixed_scale_bytes=range_scales;
+ *numeric_kinds=kinds;return 0;
}
-static int q38_segment_cache_capacity(uint64_t bytes_per_capacity,int experts,
+static int q38_segment_cache_capacity(uint64_t bytes_per_capacity,
+ uint64_t fixed_scale_bytes,int experts,
uint64_t memory_limit,int *capacity){
if(!bytes_per_capacity||experts<1||!capacity)return -1;
if(!memory_limit){*capacity=1;return 0;}
- uint64_t slots=memory_limit/bytes_per_capacity;
+ if(memory_limit<=fixed_scale_bytes)return -1;
+ uint64_t slots=(memory_limit-fixed_scale_bytes)/bytes_per_capacity;
if(!slots)return -1;
if(slots>(uint64_t)experts)slots=(uint64_t)experts;
*capacity=(int)slots;return 0;
@@ -1809,14 +1834,17 @@ static int qwen38_segment_engine_open(
* cannot discard data or perturb LRU state. */
model_init_range(&e->model,options->model_dir,1,8,
(int)e->layer_begin,(int)e->layer_end,0,0);
- uint64_t bytes_per_capacity=0;unsigned numeric_kinds=0;int cap=1;
+ uint64_t bytes_per_capacity=0,fixed_scale_bytes=0;
+ unsigned numeric_kinds=0;int cap=1;
if(q38_segment_expert_layout(&e->model,e->layer_begin,e->layer_end,
- &bytes_per_capacity,&numeric_kinds)){
+ &bytes_per_capacity,&fixed_scale_bytes,
+ &numeric_kinds)){
q38_model_free(&e->model);pthread_mutex_destroy(&e->run_lock);free(e);
return coli_segment_adapter_error(error,error_size,
"Qwen3.8 Segment checkpoint has an unsupported expert layout");
}
- if(q38_segment_cache_capacity(bytes_per_capacity,e->model.c.experts,
+ if(q38_segment_cache_capacity(bytes_per_capacity,fixed_scale_bytes,
+ e->model.c.experts,
options->memory_limit_bytes,&cap)){
q38_model_free(&e->model);pthread_mutex_destroy(&e->run_lock);free(e);
return coli_segment_adapter_error(error,error_size,
diff --git a/c/qwen38_core.h b/c/qwen38_core.h
index e6bf42f1..67e09c61 100644
--- a/c/qwen38_core.h
+++ b/c/qwen38_core.h
@@ -40,6 +40,7 @@ typedef struct {
int rows, cols;
int64_t elements, scale_count;
Q38WeightKind kind;
+ unsigned owns_data:1, owns_scales:1;
} Q38Weight;
typedef struct { float *norm; Q38Weight down, up, inject; } GatedResidual;
@@ -84,9 +85,21 @@ typedef struct {
float *ple_norm_conv, *ple_conv;
} Layer;
-typedef struct { int eid; Q38Weight gate, up, down; uint64_t used; } Slot;
+typedef struct {
+ int eid;
+ Q38Weight gate, up, down;
+ uint64_t used;
+ void *fp8_slab;
+ int64_t fp8_slab_bytes;
+} Slot;
typedef struct { Slot *slots; int *by_expert, n, cap; } LCache;
+typedef struct {
+ float *values; /* [expert][gate,up,down][block] */
+ int64_t scale_count;
+ int ready; /* 0 unknown, 1 resident, -1 incompatible */
+} Q38ExpertScaleCache;
+
typedef struct {
Cfg c;
shards S;
@@ -95,7 +108,11 @@ typedef struct {
GatedResidual final_gr;
Layer *L;
LCache *cache;
+ Q38ExpertScaleCache *expert_scales;
uint64_t clock, hits, miss;
+ uint64_t expert_weight_reads, expert_scale_reads, expert_pair_reads;
+ uint64_t expert_prefetch_ranges, expert_parallel_batches;
+ uint64_t expert_scale_bytes;
float **DN_rec, **DN_conv;
float **K, **V, **IK;
int kv_len, kv_cap, max_t;
@@ -109,7 +126,7 @@ typedef struct {
float *PLE_conv_state;
int ple_history_len;
int range_begin, range_end;
- int native_fp8, native_bf16;
+ int native_fp8, native_bf16, expert_prefetch, expert_parallel_reads;
uint64_t resident_weight_bytes;
double dense_load_s;
Q38Timers timers;
@@ -167,7 +184,9 @@ static void q38_matmul(float *y, const float *x, const float *W, int S, int I, i
static void q38_weight_free(Q38Weight *weight) {
if(!weight)return;
- free(weight->data);free(weight->scales);memset(weight,0,sizeof(*weight));
+ if(weight->owns_data)free(weight->data);
+ if(weight->owns_scales)free(weight->scales);
+ memset(weight,0,sizeof(*weight));
}
static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,int cols) {
@@ -177,7 +196,8 @@ static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,int
int64_t elements=(int64_t)rows*cols;
int64_t scales=kind==Q38_WEIGHT_FP8?fp8_nblk(rows)*fp8_nblk(cols):0;
if(weight->kind==kind&&weight->rows==rows&&weight->cols==cols&&
- weight->data&&(!scales||weight->scales))return;
+ weight->owns_data&&weight->data&&
+ (!scales||(weight->owns_scales&&weight->scales)))return;
q38_weight_free(weight);
size_t element_size=kind==Q38_WEIGHT_F32?sizeof(float):
kind==Q38_WEIGHT_BF16?sizeof(uint16_t):
@@ -194,6 +214,7 @@ static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,int
}
weight->rows=rows;weight->cols=cols;weight->elements=elements;
weight->scale_count=scales;weight->kind=kind;
+ weight->owns_data=1;weight->owns_scales=scales?1u:0u;
}
static uint64_t q38_weight_bytes(const Q38Weight *weight) {
@@ -623,6 +644,8 @@ static void model_init_range(Model *m,const char *snap,int cap,int bits,
(void)bits; memset(m,0,sizeof(*m)); double t0=now_s();
m->native_fp8=q38_env_bool("Q38_NATIVE_FP8",1);
m->native_bf16=q38_env_bool("Q38_NATIVE_BF16",1);
+ m->expert_prefetch=q38_env_bool("Q38_EXPERT_PREFETCH",1);
+ m->expert_parallel_reads=q38_env_bool("Q38_EXPERT_PARALLEL_READS",1);
q38_load_cfg(&m->c,snap); q38_validate_cfg(&m->c); st_init(&m->S,snap);
Cfg *c=&m->c; char nm[320];
if(st_has(&m->S,"model.language_model.embed_tokens.weight")) snprintf(m->prefix,sizeof m->prefix,"model.language_model");
@@ -638,7 +661,9 @@ static void model_init_range(Model *m,const char *snap,int cap,int bits,
}
m->L=(Layer*)calloc((size_t)c->layers,sizeof(Layer));
m->cache=(LCache*)calloc((size_t)c->layers,sizeof(LCache));
- if(!m->L||!m->cache){fprintf(stderr,"OOM model metadata\n");exit(1);}
+ m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers,
+ sizeof(*m->expert_scales));
+ if(!m->L||!m->cache||!m->expert_scales){fprintf(stderr,"OOM model metadata\n");exit(1);}
for(int i=layer_begin;i<layer_end;i++){
Layer *l=&m->L[i]; q38_load_gr(m,&l->attn_gr,i,"attn_hyper_connection",1); q38_load_gr(m,&l->mlp_gr,i,"mlp_hyper_connection",1);
#define WLD(field,suf,o,in) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_weight(m,nm,(o),(in))
@@ -719,6 +744,217 @@ static void q38_gr_apply(const Cfg *c,float *hyper,const float *block,const floa
}
}
+typedef struct {
+ st_tensor *tensor;
+ int expert, projection;
+} Q38ScaleDesc;
+
+static int q38_scale_desc_cmp(const void *left,const void *right) {
+ const Q38ScaleDesc *a=(const Q38ScaleDesc*)left;
+ const Q38ScaleDesc *b=(const Q38ScaleDesc*)right;
+ if(a->tensor->off<b->tensor->off)return -1;
+ if(a->tensor->off>b->tensor->off)return 1;
+ return 0;
+}
+
+static int q38_scale_group_span(Q38ScaleDesc *desc,int count,int *fd,
+ int64_t *begin,int64_t *nbytes) {
+ if(!desc||count<1||!fd||!begin||!nbytes)return -1;
+ qsort(desc,(size_t)count,sizeof(*desc),q38_scale_desc_cmp);
+ int group_fd=desc[0].tensor->fd;
+ int64_t first=desc[0].tensor->off,cursor=first;
+ for(int i=0;i<count;i++){
+ st_tensor *tensor=desc[i].tensor;
+ if(tensor->fd!=group_fd||tensor->off!=cursor||tensor->nbytes<=0||
+ tensor->nbytes>INT64_MAX-cursor)return -1;
+ cursor+=tensor->nbytes;
+ }
+ *fd=group_fd;*begin=first;*nbytes=cursor-first;return 0;
+}
+
+static void q38_decode_scale_tensor(float *out,const unsigned char *raw,
+ const st_tensor *tensor) {
+ if(tensor->dtype==2)memcpy(out,raw,(size_t)tensor->nbytes);
+ else for(int64_t i=0;i<tensor->numel;i++){
+ uint16_t half;memcpy(&half,raw+(size_t)i*sizeof(half),sizeof(half));
+ out[i]=tensor->dtype==0?bf16_to_f32(half):f16_to_f32(half);
+ }
+}
+
+/* The official checkpoint stores every layer's gate/up scale sidecars as one
+ * compact range and every down sidecar as another. Normalize both ranges to
+ * an expert-indexed F32 bank once, scattering by the numeric expert id rather
+ * than the checkpoint's lexical tensor order (0, 1, 10, ...). Variants that
+ * do not satisfy the compact-range invariant simply keep the established
+ * per-matrix loader. */
+static int q38_prepare_expert_scale_bank(Model *m,int layer) {
+ Cfg *c=&m->c;
+ if(!m->native_fp8||layer<0||layer>=c->layers)return 0;
+ if(!m->expert_scales){
+ m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers,
+ sizeof(*m->expert_scales));
+ if(!m->expert_scales){fprintf(stderr,"OOM expert scale metadata\n");exit(1);}
+ }
+ Q38ExpertScaleCache *cache=&m->expert_scales[layer];
+ if(cache->ready)return cache->ready>0;
+ int64_t scale_count=fp8_nblk(c->inter)*fp8_nblk(c->hidden);
+ if(scale_count<1||(uint64_t)c->experts>SIZE_MAX/(3u*(uint64_t)scale_count*sizeof(float))){
+ fprintf(stderr,"invalid expert scale-bank geometry\n");exit(1);
+ }
+ Q38ScaleDesc *gate_up=(Q38ScaleDesc*)malloc((size_t)c->experts*2*sizeof(*gate_up));
+ Q38ScaleDesc *down=(Q38ScaleDesc*)malloc((size_t)c->experts*sizeof(*down));
+ if(!gate_up||!down){fprintf(stderr,"OOM expert scale descriptors\n");exit(1);}
+ int ngu=0,ndown=0;char suffix[192],name[320];
+ const char *projection[3]={"gate_proj","up_proj","down_proj"};
+ for(int expert=0;expert<c->experts;expert++)for(int kind=0;kind<3;kind++){
+ int rows=kind==2?c->hidden:c->inter;
+ int cols=kind==2?c->inter:c->hidden;
+ int length=snprintf(suffix,sizeof suffix,
+ "mlp.experts.%d.%s.weight_scale_inv",expert,projection[kind]);
+ if(length<0||(size_t)length>=sizeof suffix)goto incompatible;
+ q38_name(m,name,sizeof name,layer,suffix);
+ st_tensor *tensor=st_find(&m->S,name);
+ int64_t block_rows=fp8_nblk(rows),block_cols=fp8_nblk(cols);
+ if(!tensor||tensor->dtype<0||tensor->dtype>2||tensor->rank!=2||
+ tensor->shape[0]!=block_rows||tensor->shape[1]!=block_cols||
+ tensor->numel!=scale_count||
+ tensor->numel>INT64_MAX/st_dtype_esz(tensor->dtype)||
+ tensor->nbytes!=tensor->numel*st_dtype_esz(tensor->dtype))
+ goto incompatible;
+ Q38ScaleDesc item={tensor,expert,kind};
+ if(kind<2)gate_up[ngu++]=item;else down[ndown++]=item;
+ }
+ int gu_fd,down_fd;int64_t gu_begin,down_begin,gu_bytes,down_bytes;
+ if(q38_scale_group_span(gate_up,ngu,&gu_fd,&gu_begin,&gu_bytes)||
+ q38_scale_group_span(down,ndown,&down_fd,&down_begin,&down_bytes)||
+ (uint64_t)gu_bytes>SIZE_MAX||(uint64_t)down_bytes>SIZE_MAX)
+ goto incompatible;
+ float *values=(float*)malloc((size_t)c->experts*3*(size_t)scale_count*sizeof(float));
+ unsigned char *gu_raw=(unsigned char*)malloc((size_t)gu_bytes);
+ unsigned char *down_raw=(unsigned char*)malloc((size_t)down_bytes);
+ if(!values||!gu_raw||!down_raw){fprintf(stderr,"OOM resident expert scales\n");exit(1);}
+ double started=now_s();
+ st_read_range_raw_cap(&m->S,gu_fd,gu_begin,gu_bytes,gu_raw,gu_bytes,1,
+ "pread Qwen3.8 gate/up scales");
+ st_read_range_raw_cap(&m->S,down_fd,down_begin,down_bytes,down_raw,down_bytes,1,
+ "pread Qwen3.8 down scales");
+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_scale_reads+=2;
+ for(int i=0;i<ngu;i++){
+ Q38ScaleDesc *item=&gate_up[i];
+ float *dst=values+((int64_t)item->expert*3+item->projection)*scale_count;
+ q38_decode_scale_tensor(dst,gu_raw+(item->tensor->off-gu_begin),item->tensor);
+ }
+ for(int i=0;i<ndown;i++){
+ Q38ScaleDesc *item=&down[i];
+ float *dst=values+((int64_t)item->expert*3+item->projection)*scale_count;
+ q38_decode_scale_tensor(dst,down_raw+(item->tensor->off-down_begin),item->tensor);
+ }
+ free(gu_raw);free(down_raw);free(gate_up);free(down);
+ cache->values=values;cache->scale_count=scale_count;cache->ready=1;
+ m->expert_scale_bytes+=(uint64_t)c->experts*3*(uint64_t)scale_count*sizeof(float);
+ return 1;
+incompatible:
+ free(gate_up);free(down);cache->ready=-1;return 0;
+}
+
+static void q38_bind_borrowed_fp8(Q38Weight *weight,void *data,float *scales,
+ int rows,int cols) {
+ q38_weight_free(weight);
+ weight->data=data;weight->scales=scales;weight->rows=rows;weight->cols=cols;
+ weight->elements=(int64_t)rows*cols;
+ weight->scale_count=fp8_nblk(rows)*fp8_nblk(cols);
+ weight->kind=Q38_WEIGHT_FP8;
+}
+
+static void q38_bind_fp8_slot(Slot *slot,float *scales,int scale_count,
+ int hidden,int intermediate) {
+ int64_t matrix_bytes=(int64_t)hidden*intermediate;
+ if(matrix_bytes<1||matrix_bytes>INT64_MAX/3||
+ (uint64_t)(matrix_bytes*3)>SIZE_MAX||
+ scale_count!=fp8_nblk(hidden)*fp8_nblk(intermediate)){
+ fprintf(stderr,"invalid native FP8 expert slab geometry\n");exit(1);
+ }
+ q38_weight_free(&slot->gate);q38_weight_free(&slot->up);
+ q38_weight_free(&slot->down);
+ int64_t slab_bytes=matrix_bytes*3;
+ if(!slot->fp8_slab||slot->fp8_slab_bytes!=slab_bytes){
+ void *replacement=realloc(slot->fp8_slab,(size_t)slab_bytes);
+ if(!replacement){fprintf(stderr,"OOM native FP8 expert slab\n");exit(1);}
+ slot->fp8_slab=replacement;slot->fp8_slab_bytes=slab_bytes;
+ }
+ unsigned char *raw=(unsigned char*)slot->fp8_slab;
+ q38_bind_borrowed_fp8(&slot->gate,raw,scales,intermediate,hidden);
+ q38_bind_borrowed_fp8(&slot->up,raw+matrix_bytes,scales+scale_count,
+ intermediate,hidden);
+ q38_bind_borrowed_fp8(&slot->down,raw+2*matrix_bytes,scales+2*scale_count,
+ hidden,intermediate);
+}
+
+static int q38_native_fp8_expert_tensors(Model *m,int layer,int expert,
+ st_tensor *weight[3]) {
+ if(!m->native_fp8)return 0;
+ Cfg *c=&m->c;const char *projection[3]={"gate_proj","up_proj","down_proj"};
+ int rows[3]={c->inter,c->inter,c->hidden};
+ int cols[3]={c->hidden,c->hidden,c->inter};
+ char suffix[192],name[320];
+ for(int kind=0;kind<3;kind++){
+ int length=snprintf(suffix,sizeof suffix,"mlp.experts.%d.%s.weight",
+ expert,projection[kind]);
+ if(length<0||(size_t)length>=sizeof suffix)return 0;
+ q38_name(m,name,sizeof name,layer,suffix);weight[kind]=st_find(&m->S,name);
+ if(!weight[kind]||weight[kind]->dtype!=4||weight[kind]->rank!=2||
+ weight[kind]->shape[0]!=rows[kind]||weight[kind]->shape[1]!=cols[kind]||
+ weight[kind]->nbytes!=(int64_t)rows[kind]*cols[kind])return 0;
+ }
+ if(weight[0]->fd!=weight[1]->fd||
+ weight[0]->off>INT64_MAX-weight[0]->nbytes||
+ weight[1]->off!=weight[0]->off+weight[0]->nbytes)return 0;
+ return 1;
+}
+
+static void q38_load_native_fp8_ranges(Model *m,int layer,int expert,Slot *slot,
+ st_tensor *weight[3]) {
+ Cfg *c=&m->c;
+ Q38ExpertScaleCache *cache=&m->expert_scales[layer];
+ float *scales=cache->values+(int64_t)expert*3*cache->scale_count;
+ q38_bind_fp8_slot(slot,scales,(int)cache->scale_count,c->hidden,c->inter);
+ int64_t pair_bytes=weight[0]->nbytes+weight[1]->nbytes;
+ unsigned char *raw=(unsigned char*)slot->fp8_slab;
+ st_read_range_raw_cap(&m->S,weight[0]->fd,weight[0]->off,pair_bytes,
+ raw,pair_bytes,1,"pread Qwen3.8 gate/up expert");
+ st_read_range_raw_cap(&m->S,weight[2]->fd,weight[2]->off,weight[2]->nbytes,
+ raw+pair_bytes,slot->fp8_slab_bytes-pair_bytes,1,
+ "pread Qwen3.8 down expert");
+}
+
+static int q38_try_load_native_fp8_expert(Model *m,int layer,int expert,Slot *slot) {
+ st_tensor *weight[3];
+ if(!q38_native_fp8_expert_tensors(m,layer,expert,weight)||
+ !q38_prepare_expert_scale_bank(m,layer))return 0;
+ double started=now_s();
+ q38_load_native_fp8_ranges(m,layer,expert,slot,weight);
+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);
+ m->expert_weight_reads+=2;m->expert_pair_reads++;
+ return 1;
+}
+
+static void q38_prefetch_native_fp8_experts(Model *m,int layer,
+ const int *experts,int count) {
+ if(!m->expert_prefetch||!m->native_fp8||!experts||count<1||
+ !q38_prepare_expert_scale_bank(m,layer))return;
+ LCache *cache=&m->cache[layer];
+ for(int index=0;index<count;index++){
+ int expert=experts[index];st_tensor *weight[3];
+ if(expert<0||expert>=m->c.experts||cache->by_expert[expert]>=0||
+ !q38_native_fp8_expert_tensors(m,layer,expert,weight))continue;
+ posix_fadvise(weight[0]->fd,weight[0]->off,
+ weight[0]->nbytes+weight[1]->nbytes,POSIX_FADV_WILLNEED);
+ posix_fadvise(weight[2]->fd,weight[2]->off,weight[2]->nbytes,
+ POSIX_FADV_WILLNEED);
+ m->expert_prefetch_ranges+=2;
+ }
+}
+
static void q38_load_fp8_expert_weight(Model *m,const char *wn,const char *sn,
Q38Weight *out,int O,int I) {
st_tensor *w=st_find(&m->S,wn),*sc=st_find(&m->S,sn);
@@ -735,6 +971,7 @@ static void q38_load_fp8_expert_weight(Model *m,const char *wn,const char *sn,
st_read_raw_cap(&m->S,wn,out->data,w->nbytes,1);
st_read_f32(&m->S,sn,out->scales,1);
q38_tm_add(m,Q38_TM_EXPERT_READ,started);
+ m->expert_weight_reads++;m->expert_scale_reads++;
return;
}
q38_weight_reserve(out,Q38_WEIGHT_F32,O,I);
@@ -742,7 +979,8 @@ static void q38_load_fp8_expert_weight(Model *m,const char *wn,const char *sn,
if(!raw){fprintf(stderr,"OOM FP8 expert staging\n");exit(1);}
double started=now_s();
st_read_raw_cap(&m->S,wn,raw,w->nbytes,1);st_read_f32(&m->S,sn,scale,1);
- q38_tm_add(m,Q38_TM_EXPERT_READ,started); started=now_s();
+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);
+ m->expert_weight_reads++;m->expert_scale_reads++;started=now_s();
float *decoded=(float*)out->data;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++)for(int i=0;i<I;i++)
@@ -764,7 +1002,7 @@ static void q38_load_expert_weight(Model *m,const char *name,Q38Weight *out,
if(kind==Q38_WEIGHT_BF16)
st_read_raw_cap(&m->S,name,out->data,tensor->nbytes,1);
else st_read_f32(&m->S,name,(float*)out->data,1);
- q38_tm_add(m,Q38_TM_EXPERT_READ,started);
+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;
}
static void q38_load_expert_slice(Model *m,const char *name,const st_tensor *tensor,
@@ -780,7 +1018,7 @@ static void q38_load_expert_slice(Model *m,const char *name,const st_tensor *ten
st_read_slice_raw_cap(&m->S,name,element_offset*(int64_t)sizeof(uint16_t),
bytes,out->data,bytes,1);
} else st_read_slice_f32(&m->S,name,element_offset,(int64_t)O*I,(float*)out->data,1);
- q38_tm_add(m,Q38_TM_EXPERT_READ,started);
+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;
}
static void q38_load_expert(Model *m,int layer,int eid,Slot *s) {
@@ -797,6 +1035,7 @@ static void q38_load_expert(Model *m,int layer,int eid,Slot *s) {
q38_load_expert_slice(m,nm,t,(int64_t)eid*H*I,&s->down,H,I);
return;
}
+ if(q38_try_load_native_fp8_expert(m,layer,eid,s))return;
const char *kind[3]={"gate_proj","up_proj","down_proj"};Q38Weight *dst[3]={&s->gate,&s->up,&s->down};
int os[3]={I,I,H},is[3]={H,H,I};
for(int k=0;k<3;k++){
@@ -819,6 +1058,97 @@ static Slot *q38_expert_get(Model *m,int layer,int eid) {
s->eid=-1;q38_load_expert(m,layer,eid,s);s->eid=eid;s->used=++m->clock;lc->by_expert[eid]=(int)(s-lc->slots);return s;
}
+typedef struct {
+ int expert;
+ Slot *slot;
+ st_tensor *weight[3];
+} Q38ExpertLoadJob;
+
+/* Reserve an entire routed demand set on the main thread, fill distinct slots
+ * in parallel, then publish every cache index on the main thread. Demand-set
+ * residents are protected from victim selection, so no worker can overwrite a
+ * slot another selected expert will consume. Smaller caches and heterogeneous
+ * layouts retain the serial LRU path. */
+static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,
+ Slot **selected) {
+ if(!m->expert_parallel_reads||!experts||!selected||count<2||
+ count>Q38_MAX_TOPK)return 0;
+ LCache *cache=&m->cache[layer];
+ if(cache->cap<count||!q38_prepare_expert_scale_bank(m,layer))return 0;
+ Q38ExpertLoadJob jobs[Q38_MAX_TOPK];
+ for(int index=0;index<count;index++){
+ int expert=experts[index];
+ if(expert<0||expert>=m->c.experts)return 0;
+ for(int previous=0;previous<index;previous++)
+ if(experts[previous]==expert)return 0;
+ int slot_index=cache->by_expert[expert];
+ if(slot_index>=0){
+ if(slot_index>=cache->n||cache->slots[slot_index].eid!=expert)return 0;
+ continue;
+ }
+ st_tensor *weight[3];
+ if(!q38_native_fp8_expert_tensors(m,layer,expert,weight))return 0;
+ }
+ unsigned char *protected_slots=(unsigned char*)calloc((size_t)cache->cap,1);
+ if(!protected_slots){fprintf(stderr,"OOM expert batch reservations\n");exit(1);}
+ for(int index=0;index<count;index++){
+ int slot_index=cache->by_expert[experts[index]];
+ if(slot_index>=0)protected_slots[slot_index]=1;
+ }
+ int job_count=0;
+ for(int index=0;index<count;index++){
+ int expert=experts[index],slot_index=cache->by_expert[expert];Slot *slot;
+ if(slot_index>=0){
+ slot=&cache->slots[slot_index];m->hits++;
+ }else{
+ m->miss++;
+ if(cache->n<cache->cap){
+ slot=&cache->slots[cache->n++];slot->eid=-1;
+ }else{
+ int victim=-1;
+ for(int candidate=0;candidate<cache->n;candidate++)
+ if(!protected_slots[candidate]&&
+ (victim<0||cache->slots[candidate].used<cache->slots[victim].used))
+ victim=candidate;
+ if(victim<0){
+ fprintf(stderr,"Qwen3.8 expert demand set has no reservable cache slot\n");
+ exit(1);
+ }
+ slot=&cache->slots[victim];
+ if(slot->eid>=0)cache->by_expert[slot->eid]=-1;
+ }
+ slot->eid=-1;jobs[job_count].expert=expert;jobs[job_count].slot=slot;
+ if(!q38_native_fp8_expert_tensors(m,layer,expert,jobs[job_count].weight)){
+ fprintf(stderr,"Qwen3.8 expert layout changed during batch reservation\n");exit(1);
+ }
+ job_count++;
+ }
+ slot->used=++m->clock;selected[index]=slot;
+ protected_slots[slot-cache->slots]=1;
+ }
+ free(protected_slots);
+ if(job_count){
+ int workers=job_count;
+#ifdef _OPENMP
+ int thread_limit=omp_get_max_threads();if(workers>thread_limit)workers=thread_limit;
+#endif
+ double started=now_s();
+ #pragma omp parallel for schedule(static) num_threads(workers) if(job_count>1)
+ for(int job=0;job<job_count;job++)
+ q38_load_native_fp8_ranges(m,layer,jobs[job].expert,jobs[job].slot,
+ jobs[job].weight);
+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);
+ m->expert_weight_reads+=(uint64_t)job_count*2;
+ m->expert_pair_reads+=(uint64_t)job_count;
+ if(job_count>1)m->expert_parallel_batches++;
+ for(int job=0;job<job_count;job++){
+ Slot *slot=jobs[job].slot;slot->eid=jobs[job].expert;
+ cache->by_expert[jobs[job].expert]=(int)(slot-cache->slots);
+ }
+ }
+ return 1;
+}
+
static void q38_ple_row(Model *m,int64_t row,float *out) {
Cfg *c=&m->c; int p=0;
while(p+1<m->ple_part_count&&row>=m->ple_part_start[p+1])p++;
@@ -990,19 +1320,22 @@ static void q38_moe(Model *m,Layer *l,int layer,const float *x,int S,float *out)
float route_gates[Q38_MAX_TOPK];
for(int z=0;z<K;z++) route_gates[z]=(float)(val[z]/den);
rt_route(layer,s,idx,route_gates,K); /* shared counts + post-normalization trace */
+ q38_prefetch_native_fp8_experts(m,layer,idx,K);
+ double phase_started=now_s();
+ q38_weight_matmul(sg,xs,&l->sh_g,1,H,SI);q38_weight_matmul(su,xs,&l->sh_u,1,H,SI);
+ for(int j=0;j<SI;j++)sh[j]=q38_silu(sg[j])*su[j];q38_weight_matmul(shared,sh,&l->sh_d,1,SI,H);
+ float gate=0.f;for(int d=0;d<H;d++)gate+=xs[d]*l->sh_gate[d];gate=q38_sigmoid(gate);
+ q38_tm_add(m,Q38_TM_SHARED_EXPERT,phase_started);
+ Slot *selected[Q38_MAX_TOPK];
+ int loaded_batch=q38_expert_get_batch(m,layer,idx,K,selected);
for(int z=0;z<K;z++){
- Slot *ex=q38_expert_get(m,layer,idx[z]); double phase_started=now_s();
+ Slot *ex=loaded_batch?selected[z]:q38_expert_get(m,layer,idx[z]);phase_started=now_s();
q38_weight_matmul(eg,xs,&ex->gate,1,H,I);q38_weight_matmul(eu,xs,&ex->up,1,H,I);
for(int j=0;j<I;j++)eh[j]=q38_silu(eg[j])*eu[j];q38_weight_matmul(eo,eh,&ex->down,1,I,H);
for(int d=0;d<H;d++)ys[d]+=route_gates[z]*eo[d];
q38_tm_add(m,Q38_TM_ROUTED_EXPERT,phase_started);
}
- double phase_started=now_s();
- q38_weight_matmul(sg,xs,&l->sh_g,1,H,SI);q38_weight_matmul(su,xs,&l->sh_u,1,H,SI);
- for(int j=0;j<SI;j++)sh[j]=q38_silu(sg[j])*su[j];q38_weight_matmul(shared,sh,&l->sh_d,1,SI,H);
- float gate=0.f;for(int d=0;d<H;d++)gate+=xs[d]*l->sh_gate[d];gate=q38_sigmoid(gate);
for(int d=0;d<H;d++)ys[d]+=gate*shared[d];
- q38_tm_add(m,Q38_TM_SHARED_EXPERT,phase_started);
}
rt_trace_end();
free(logits);free(sg);free(su);free(sh);free(shared);free(eg);free(eu);free(eh);free(eo);
@@ -1101,6 +1434,16 @@ static void q38_tm_report_bank(const Q38Timers *timers,const char *scope) {
static void tm_report(const Model *m) {
q38_tm_report_bank(&m->timers,"total");
+ if(q38_tm_enabled())
+ fprintf(stderr,"[qwen38 expert I/O] weight-ranges=%llu scale-ranges=%llu "
+ "coalesced-gate-up=%llu prefetched=%llu parallel-batches=%llu "
+ "resident-scales=%.2f MiB\n",
+ (unsigned long long)m->expert_weight_reads,
+ (unsigned long long)m->expert_scale_reads,
+ (unsigned long long)m->expert_pair_reads,
+ (unsigned long long)m->expert_prefetch_ranges,
+ (unsigned long long)m->expert_parallel_batches,
+ m->expert_scale_bytes/1048576.0);
}
static void q38_layer_free(Layer *l) {
@@ -1126,13 +1469,15 @@ static void q38_model_free(Model *m) {
q38_weight_free(&m->cache[i].slots[s].gate);
q38_weight_free(&m->cache[i].slots[s].up);
q38_weight_free(&m->cache[i].slots[s].down);
+ free(m->cache[i].slots[s].fp8_slab);
}
free(m->cache[i].slots); free(m->cache[i].by_expert);
}
+ if(m->expert_scales)free(m->expert_scales[i].values);
free(m->DN_rec ? m->DN_rec[i] : NULL); free(m->DN_conv ? m->DN_conv[i] : NULL);
free(m->K ? m->K[i] : NULL); free(m->V ? m->V[i] : NULL); free(m->IK ? m->IK[i] : NULL);
}
- free(m->L); free(m->cache); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);
+ free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);
q38_weight_free(&m->embed);q38_weight_free(&m->lm_head);
free(m->final_gr.norm);q38_weight_free(&m->final_gr.down);q38_weight_free(&m->final_gr.up);q38_weight_free(&m->final_gr.inject);
free(m->ple_history); free(m->PLE_conv_state); free(m->c.is_attn); st_destroy(&m->S);
diff --git a/c/st.h b/c/st.h
index bfdea716..9f6a55f3 100644
--- a/c/st.h
+++ b/c/st.h
@@ -938,6 +938,30 @@ static void st_read_raw_cap(shards *S, const char *name, void *out, int64_t cap,
st_read_raw(S, name, out, drop);
}
+/* Read an exact physical range from one indexed shard. This is intentionally
+ * separate from tensor slices: callers may use it only after validating a
+ * checkpoint-specific packing invariant (for example, two adjacent tensors).
+ * The shard membership, file bounds and destination capacity remain enforced
+ * here so a malformed header cannot turn that optimization into an unchecked
+ * pread. */
+static void st_read_range_raw_cap(shards *S, int fd, int64_t off,
+ int64_t nbytes, void *out, int64_t cap,
+ int drop, const char *tag) {
+ int fidx=S?st_fidx(S,fd):-1;
+ if(fidx<0){fprintf(stderr,"physical range uses an unindexed shard fd\n");exit(1);}
+ if(off<0||nbytes<0||cap<0||nbytes>cap||
+ off>S->sizes[fidx]||nbytes>S->sizes[fidx]-off||
+ (uint64_t)nbytes>SIZE_MAX){
+ fprintf(stderr,"physical shard range [%lld,+%lld) exceeds file/destination bounds "
+ "(file %lld, cap %lld) — refusing (untrusted container)\n",
+ (long long)off,(long long)nbytes,(long long)S->sizes[fidx],
+ (long long)cap);exit(1);
+ }
+ if(nbytes>0&&!out){fprintf(stderr,"physical range has a NULL destination\n");exit(1);}
+ if(nbytes>0)st_pread_full(fd,out,nbytes,off,tag&&*tag?tag:"pread physical range");
+ if(drop&&nbytes>0)posix_fadvise(fd,off,nbytes,POSIX_FADV_DONTNEED);
+}
+
/* Read-only view of one tensor's exact stored bytes. Unlike st_read_raw this
* performs no allocation or copy; unlike a naked mmap pointer it carries the
* aligned OS view/handle required for cleanup. Callers must still validate
diff --git a/c/tests/test_qwen38_native_weights.c b/c/tests/test_qwen38_native_weights.c
index eaadb18f..9889ded6 100644
--- a/c/tests/test_qwen38_native_weights.c
+++ b/c/tests/test_qwen38_native_weights.c
@@ -20,7 +20,7 @@ typedef struct {
unsigned char raw[24];
size_t raw_bytes;
float scale;
- int is_scale;
+ int expert,projection,is_scale;
} FixtureTensor;
static uint16_t to_bf16(float value){
@@ -61,10 +61,12 @@ static int add_fixture_tensor(FixtureTensor *tensors,int *count,int64_t *offset,
int length=snprintf(tensor->name,sizeof tensor->name,"%s",name);
if(length<0||(size_t)length>=sizeof tensor->name)return -1;
tensor->dtype=dtype;tensor->rows=rows;tensor->cols=cols;
- tensor->begin=*offset;tensor->is_scale=is_scale;
+ tensor->begin=*offset;tensor->expert=expert;tensor->projection=projection;
+ tensor->is_scale=is_scale;
if(is_scale){
tensor->scale=0.125f*(float)(1+expert*3+projection);
- *offset+=(int64_t)sizeof(float);
+ *offset+=!strcmp(dtype,"F32")?(int64_t)sizeof(float):
+ (int64_t)sizeof(uint16_t);
}else if(!strcmp(dtype,"BF16")){
tensor->raw_bytes=(size_t)rows*cols*sizeof(uint16_t);
for(int index=0;index<rows*cols;index++){
@@ -81,7 +83,7 @@ static int add_fixture_tensor(FixtureTensor *tensors,int *count,int64_t *offset,
tensor->end=*offset;return 0;
}
-static int write_fp8_fixture(const char *directory,int mixed){
+static int write_fp8_fixture(const char *directory,int mixed,int adjacent){
FixtureTensor tensors[12];int count=0;int64_t offset=0;
const char *projection[3]={"gate_proj","up_proj","down_proj"};
for(int expert=0;expert<2;expert++)for(int kind=0;kind<3;kind++){
@@ -97,10 +99,41 @@ static int write_fp8_fixture(const char *directory,int mixed){
"model.layers.0.mlp.experts.%d.%s.weight_scale_inv",expert,
projection[kind]);
if(length<0||(size_t)length>=sizeof name||
- add_fixture_tensor(tensors,&count,&offset,name,"F32",1,1,
+ add_fixture_tensor(tensors,&count,&offset,name,adjacent?"BF16":"F32",1,1,
expert,kind,1))return -1;
}
+ int order[12];for(int index=0;index<count;index++)order[index]=index;
+ if(adjacent){
+ /* Match the official packing invariant while deliberately keeping the
+ * JSON entries in construction order: compact gate/up sidecars, compact
+ * down sidecars, then gate+up payload pairs. The production loader
+ * must use offsets and numeric expert ids, not header order. */
+ for(int left=1;left<count;left++){
+ int value=order[left],right=left-1;
+ FixtureTensor *v=&tensors[value];
+ int vcat=v->is_scale?(v->projection<2?0:1):2;
+ int vkey=vcat*100+v->expert*3+v->projection;
+ while(right>=0){
+ FixtureTensor *r=&tensors[order[right]];
+ int rcat=r->is_scale?(r->projection<2?0:1):2;
+ int rkey=rcat*100+r->expert*3+r->projection;
+ if(rkey<=vkey)break;
+ order[right+1]=order[right];right--;
+ }
+ order[right+1]=value;
+ }
+ offset=0;
+ for(int physical=0;physical<count;physical++){
+ FixtureTensor *tensor=&tensors[order[physical]];
+ int64_t bytes=tensor->is_scale?
+ (!strcmp(tensor->dtype,"F32")?(int64_t)sizeof(float):
+ (int64_t)sizeof(uint16_t)):
+ (int64_t)tensor->raw_bytes;
+ tensor->begin=offset;tensor->end=offset+bytes;offset+=bytes;
+ }
+ }
+
char header[8192];size_t used=0;header[used++]='{';
for(int index=0;index<count;index++){
FixtureTensor *tensor=&tensors[index];
@@ -121,10 +154,14 @@ static int write_fp8_fixture(const char *directory,int mixed){
uint64_t header_length=used;
int failed=fwrite(&header_length,sizeof header_length,1,file)!=1||
fwrite(header,1,used,file)!=used;
- for(int index=0;index<count&&!failed;index++){
- FixtureTensor *tensor=&tensors[index];
- if(tensor->is_scale)
+ for(int physical=0;physical<count&&!failed;physical++){
+ FixtureTensor *tensor=&tensors[order[physical]];
+ if(tensor->is_scale&&!strcmp(tensor->dtype,"F32"))
failed=fwrite(&tensor->scale,sizeof tensor->scale,1,file)!=1;
+ else if(tensor->is_scale){
+ uint16_t scale=to_bf16(tensor->scale);
+ failed=fwrite(&scale,sizeof scale,1,file)!=1;
+ }
else
failed=fwrite(tensor->raw,1,tensor->raw_bytes,file)!=tensor->raw_bytes;
}
@@ -155,10 +192,13 @@ static void destroy_fixture_model(Model *model){
q38_weight_free(&model->cache[0].slots[slot].gate);
q38_weight_free(&model->cache[0].slots[slot].up);
q38_weight_free(&model->cache[0].slots[slot].down);
+ free(model->cache[0].slots[slot].fp8_slab);
}
free(model->cache[0].slots);free(model->cache[0].by_expert);
}
- free(model->cache);st_destroy(&model->S);memset(model,0,sizeof(*model));
+ if(model->expert_scales)free(model->expert_scales[0].values);
+ free(model->expert_scales);free(model->cache);st_destroy(&model->S);
+ memset(model,0,sizeof(*model));
}
static uint8_t fixture_fp8_byte(int expert,int projection,int index){
@@ -204,25 +244,30 @@ static int verify_loaded_fp8_slot(const Slot *slot,int expert,int native_fp8){
verify_loaded_fp8_weight(&slot->down,expert,2,native_fp8)?-1:0;
}
-static int check_fixture_mode(const char *directory,int native_fp8){
+static int check_fixture_mode(const char *directory,int native_fp8,int expect_fast){
Model model;if(init_fixture_model(&model,directory,native_fp8))return -1;
int result=-1;
- uint64_t bytes_per_capacity=0;unsigned numeric_kinds=0;
- if(q38_segment_expert_layout(&model,0,1,&bytes_per_capacity,&numeric_kinds))
+ uint64_t bytes_per_capacity=0,fixed_scale_bytes=0;unsigned numeric_kinds=0;
+ if(q38_segment_expert_layout(&model,0,1,&bytes_per_capacity,
+ &fixed_scale_bytes,&numeric_kinds))
goto cleanup;
char numeric_class[96];
q38_segment_numeric_class(numeric_class,sizeof numeric_class,numeric_kinds);
if(native_fp8){
- if(bytes_per_capacity!=30||numeric_kinds!=Q38_EXPERT_FP8_BLOCK||
+ if(bytes_per_capacity!=18||fixed_scale_bytes!=24||
+ numeric_kinds!=Q38_EXPERT_FP8_BLOCK||
strcmp(numeric_class,
"qwen38/fp8-block-f32dot-f64blocksum/cpu-v1"))goto cleanup;
- }else if(bytes_per_capacity!=72||numeric_kinds!=Q38_EXPERT_FP8_EXPANDED||
+ }else if(bytes_per_capacity!=72||fixed_scale_bytes||
+ numeric_kinds!=Q38_EXPERT_FP8_EXPANDED||
strcmp(numeric_class,
"qwen38/fp8-expanded-f32dot/cpu-v1"))goto cleanup;
int capacity=0;
- if(q38_segment_cache_capacity(bytes_per_capacity,2,bytes_per_capacity*2,
+ if(q38_segment_cache_capacity(bytes_per_capacity,fixed_scale_bytes,2,
+ fixed_scale_bytes+bytes_per_capacity*2,
&capacity)||capacity!=2||
- !q38_segment_cache_capacity(bytes_per_capacity,2,bytes_per_capacity-1,
+ !q38_segment_cache_capacity(bytes_per_capacity,fixed_scale_bytes,2,
+ fixed_scale_bytes+bytes_per_capacity-1,
&capacity))goto cleanup;
if(q38_segment_cache_resize(&model,2)||model.cache[0].cap!=2||
q38_segment_cache_resize(&model,1)||model.cache[0].cap!=1)goto cleanup;
@@ -232,6 +277,15 @@ static int check_fixture_mode(const char *directory,int native_fp8){
if(first->gate.kind!=expected||first->up.kind!=expected||
first->down.kind!=expected||model.miss!=1||model.hits!=0||
verify_loaded_fp8_slot(first,0,native_fp8))goto cleanup;
+ void *slab=first->fp8_slab;
+ if(expect_fast){
+ if(!native_fp8||!slab||first->gate.data!=slab||
+ first->up.data!=(unsigned char*)slab+6||
+ first->down.data!=(unsigned char*)slab+12||
+ model.expert_weight_reads!=2||model.expert_scale_reads!=2||
+ model.expert_pair_reads!=1||model.expert_scale_bytes!=24)
+ goto cleanup;
+ }else if(model.expert_pair_reads||slab)goto cleanup;
float first_value=native_fp8?e4m3_decode(((uint8_t*)first->gate.data)[0])*
first->gate.scales[0]:
((float*)first->gate.data)[0];
@@ -240,12 +294,18 @@ static int check_fixture_mode(const char *directory,int native_fp8){
if(second!=first||model.cache[0].by_expert[0]!=-1||
model.cache[0].by_expert[1]!=0||model.miss!=2||model.hits!=1||
verify_loaded_fp8_slot(second,1,native_fp8))goto cleanup;
+ if(expect_fast&&(second->fp8_slab!=slab||model.expert_weight_reads!=4||
+ model.expert_scale_reads!=2||model.expert_pair_reads!=2))
+ goto cleanup;
float second_value=native_fp8?e4m3_decode(((uint8_t*)second->gate.data)[0])*
second->gate.scales[0]:
((float*)second->gate.data)[0];
if(first_value==second_value)goto cleanup;
if(q38_expert_get(&model,0,0)!=first||model.cache[0].by_expert[0]!=0||
model.cache[0].by_expert[1]!=-1||model.miss!=3)goto cleanup;
+ if(expect_fast&&(first->fp8_slab!=slab||model.expert_weight_reads!=6||
+ model.expert_scale_reads!=2||model.expert_pair_reads!=3))
+ goto cleanup;
result=0;
cleanup:
destroy_fixture_model(&model);return result;
@@ -255,10 +315,10 @@ static int check_mixed_layout(const char *directory,int native_fp8,
uint64_t expected_bytes,unsigned expected_kinds,
const char *expected_class){
Model model;if(init_fixture_model(&model,directory,native_fp8))return -1;
- uint64_t bytes=0;unsigned kinds=0;char numeric_class[96];int result=-1;
- if(!q38_segment_expert_layout(&model,0,1,&bytes,&kinds)){
+ uint64_t bytes=0,fixed=0;unsigned kinds=0;char numeric_class[96];int result=-1;
+ if(!q38_segment_expert_layout(&model,0,1,&bytes,&fixed,&kinds)){
q38_segment_numeric_class(numeric_class,sizeof numeric_class,kinds);
- if(bytes==expected_bytes&&kinds==expected_kinds&&
+ if(bytes==expected_bytes&&!fixed&&kinds==expected_kinds&&
!strcmp(numeric_class,expected_class)){
Slot *slot=q38_expert_get(&model,0,0);
if(slot&&!verify_loaded_fp8_slot(slot,0,native_fp8)&&
@@ -284,6 +344,29 @@ static int check_mixed_layout(const char *directory,int native_fp8,
destroy_fixture_model(&model);return result;
}
+static int check_parallel_batch(const char *directory){
+ Model model;if(init_fixture_model(&model,directory,1))return -1;
+ int result=-1;model.expert_parallel_reads=1;
+ int first_ids[2]={1,0};Slot *first[2]={0};
+ if(q38_expert_get_batch(&model,0,first_ids,2,first)||model.cache[0].n||
+ model.miss||model.hits)goto cleanup;
+ if(q38_segment_cache_resize(&model,2))goto cleanup;
+ if(!q38_expert_get_batch(&model,0,first_ids,2,first)||!first[0]||!first[1]||
+ first[0]==first[1]||first[0]->eid!=1||first[1]->eid!=0||
+ model.miss!=2||model.hits||model.expert_parallel_batches!=1||
+ model.expert_weight_reads!=4||model.expert_scale_reads!=2||
+ model.expert_pair_reads!=2||verify_loaded_fp8_slot(first[0],1,1)||
+ verify_loaded_fp8_slot(first[1],0,1))goto cleanup;
+ int second_ids[2]={0,1};Slot *second[2]={0};
+ if(!q38_expert_get_batch(&model,0,second_ids,2,second)||
+ second[0]!=first[1]||second[1]!=first[0]||model.miss!=2||
+ model.hits!=2||model.expert_parallel_batches!=1||
+ model.expert_weight_reads!=4)goto cleanup;
+ result=0;
+cleanup:
+ destroy_fixture_model(&model);return result;
+}
+
static int check_segment_failure_outputs(void){
char error[128];void *output=(void*)(uintptr_t)1;
ColiSegmentCapabilities capabilities;
@@ -339,9 +422,9 @@ int main(void){
q38_weight_free(&bf16);
char directory[]="test_qwen38_native_XXXXXX";CHECK(mkdtemp(directory)!=NULL);
- CHECK(write_fp8_fixture(directory,0)==0);
- CHECK(check_fixture_mode(directory,1)==0);
- CHECK(check_fixture_mode(directory,0)==0);
+ CHECK(write_fp8_fixture(directory,0,0)==0);
+ CHECK(check_fixture_mode(directory,1,0)==0);
+ CHECK(check_fixture_mode(directory,0,0)==0);
char path[512];int path_length=snprintf(path,sizeof path,
"%s/model.safetensors",directory);
CHECK(path_length>0&&(size_t)path_length<sizeof path);
@@ -349,7 +432,7 @@ int main(void){
char mixed_directory[]="test_qwen38_mixed_XXXXXX";
CHECK(mkdtemp(mixed_directory)!=NULL);
- CHECK(write_fp8_fixture(mixed_directory,1)==0);
+ CHECK(write_fp8_fixture(mixed_directory,1,0)==0);
CHECK(check_mixed_layout(mixed_directory,1,32,
Q38_EXPERT_FP8_BLOCK|Q38_EXPERT_BF16,
"qwen38/mixed-expert-arithmetic-05/cpu-v1")==0);
@@ -360,8 +443,18 @@ int main(void){
CHECK(path_length>0&&(size_t)path_length<sizeof path);
CHECK(remove(path)==0);CHECK(rmdir(mixed_directory)==0);
+ char adjacent_directory[]="test_qwen38_adjacent_XXXXXX";
+ CHECK(mkdtemp(adjacent_directory)!=NULL);
+ CHECK(write_fp8_fixture(adjacent_directory,0,1)==0);
+ CHECK(check_fixture_mode(adjacent_directory,1,1)==0);
+ CHECK(check_fixture_mode(adjacent_directory,0,0)==0);
+ CHECK(check_parallel_batch(adjacent_directory)==0);
+ path_length=snprintf(path,sizeof path,"%s/model.safetensors",adjacent_directory);
+ CHECK(path_length>0&&(size_t)path_length<sizeof path);
+ CHECK(remove(path)==0);CHECK(rmdir(adjacent_directory)==0);
+
CHECK(check_segment_failure_outputs()==0);
- puts("qwen38 native weights: independent FP8 oracle, real loader, LRU, mixed classes, adapter failures: ok");
+ puts("qwen38 native weights: FP8 oracle, coalesced slabs/scales, parallel demand sets, fallbacks, adapter failures: ok");
return 0;
}
diff --git a/c/tests/test_qwen38_tokenizer.c b/c/tests/test_qwen38_tokenizer.c
index 393e50f1..c851cf41 100644
--- a/c/tests/test_qwen38_tokenizer.c
+++ b/c/tests/test_qwen38_tokenizer.c
@@ -160,9 +160,9 @@ int main(void) {
uint8_t layer_kinds[2]={0,1};Cfg cfg={0};cfg.layers=2;cfg.is_attn=layer_kinds;
cfg.hidden=10;cfg.inter=2;cfg.experts=9;
int cache_capacity=0;
- CHECK(q38_segment_cache_capacity(480,cfg.experts,960,&cache_capacity)==0);
+ CHECK(q38_segment_cache_capacity(480,0,cfg.experts,960,&cache_capacity)==0);
CHECK(cache_capacity==2);
- CHECK(q38_segment_cache_capacity(480,cfg.experts,479,&cache_capacity)<0);
+ CHECK(q38_segment_cache_capacity(480,0,cfg.experts,479,&cache_capacity)<0);
cfg.dn_vheads=2;cfg.dn_kdim=3;cfg.dn_vdim=4;cfg.dn_conv_dim=5;cfg.dn_convk=3;
cfg.kv_heads=2;cfg.head_dim=4;cfg.idx_dim=3;
cfg.ple_layer=0;cfg.hc_width=8;cfg.ple_convk=3;cfg.ngram_size=3;