Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen38: normalize core source line endings
Stamina9cf8ad45528ba
diff --git a/c/qwen38_core.h b/c/qwen38_core.hindex 25ec0b56..f68cb4f8 100644--- a/c/qwen38_core.h+++ b/c/qwen38_core.h@@ -1,109 +1,109 @@-#include "qwen36_tier.h" /* CUDA VRAM expert tier, shared with qwen36; CPU-only builds get the inline no-op stubs */-/* Native Qwen3.8-Flash-Next text core.- *- * This header is included once by qwen38.c after its tokenizer and protocol- * helpers. It intentionally consumes the official HF safetensors layout:- * the multimodal wrapper's `model.language_model` namespace and the standalone- * text model's `model` namespace are both accepted. Vision and MTP tensors are- * indexed by st.h but are never read.- */-#ifndef COLI_QWEN38_CORE_H-#define COLI_QWEN38_CORE_H-#include <pthread.h> /* q38_ehit_mark publishes the lazy HITS table under a lock */--#define Q38_MAX_LAYERS 512-#define Q38_MAX_EXPERTS 1024-#define Q38_MAX_TOPK 256-#define Q38_MAX_PLE_PARTS 512-#define Q38_PREFILL_BATCH_ROWS 32-#define Q38_PREFILL_WORKSPACE_BYTES (64u << 20)--typedef struct {- int hidden, layers, vocab, max_positions, eos_id;- float eps, theta;- int hc_count, hc_rank, hc_width;- int q_heads, kv_heads, head_dim, rotary_dim;- int idx_qheads, idx_kheads, idx_dim, idx_budget, idx_ratio;- int experts, topk, inter, shared_inter, norm_topk;- int dn_kheads, dn_vheads, dn_kdim, dn_vdim, dn_convk, dn_conv_dim;- int ple_layer, ple_dim, ple_convk, ngram_size, heads_per_ngram;- int ngram_heads, ngram_head_dim, ngram_parts;- uint8_t *is_attn;- /* vision: 0 = checkpoint di solo testo, o torre non caricata */- int image_token;- int vis_depth, vis_hidden, vis_heads, vis_inter, vis_patch;- int vis_merge, vis_temporal, vis_in_ch, vis_out_hidden, vis_num_pos;-} Cfg;--typedef enum {- Q38_WEIGHT_NONE = 0,- Q38_WEIGHT_F32,- Q38_WEIGHT_BF16,- Q38_WEIGHT_FP8-} Q38WeightKind;--typedef struct {- void *data;- float *scales; /* block-FP8 only */- int rows, cols;- int64_t elements, scale_count;- Q38WeightKind kind;- unsigned owns_data:1, owns_scales:1;- int gpu; /* 0 = CPU; else 1 + tier handle of an int8 copy resident in VRAM (decode, S == 1) */- int8_t *q8; float *q8sc; /* Q38_TRUNK_CPU_INT8=1: the same int8 rows kept on the CPU (reference for the GPU path, no GPU needed) */-} Q38Weight;--typedef struct { float *norm; Q38Weight down, up, inject; } GatedResidual;--/* Persistent wall-clock counters. They live on Model rather than in process- * globals so Segment sessions and future multi-model serving cannot leak phase- * time into one another. Some categories intentionally overlap: architecture- * phases (DeltaNet/QSA/PLE) contain resident matmuls, while the matmul counter- * answers the orthogonal question "how much time is in dense kernels?". */-typedef enum {- Q38_TM_EXPERT_READ = 0,- Q38_TM_FP8_EXPAND,- Q38_TM_ROUTED_EXPERT,- Q38_TM_SHARED_EXPERT,- Q38_TM_DENSE_MATMUL,- Q38_TM_DELTANET,- Q38_TM_QSA_INDEX,- Q38_TM_QSA_ATTENTION,- Q38_TM_PLE,- Q38_TM_LM_HEAD,- Q38_TM_COUNT-} Q38Timer;--typedef struct {- double seconds[Q38_TM_COUNT];- uint64_t forwards;-} Q38Timers;--typedef struct {- GatedResidual attn_gr, mlp_gr;- Q38Weight router, sh_g, sh_u, sh_d;- float *sh_gate;- Q38Weight q, k, v, o;- float *qn, *kn;- Q38Weight idx_qk;- float *idx_qn, *idx_kn;- Q38Weight dn_qkv, dn_z, dn_b, dn_a, dn_out;- float *dn_conv;- float *dn_dtbias, *dn_alog, *dn_norm;- Q38Weight ple_key, ple_value;- float *ple_norm_key, *ple_norm_query;- float *ple_norm_conv, *ple_conv;-} Layer;--typedef struct {- int eid;- Q38Weight gate, up, down;- uint64_t used;- void *fp8_slab;- int64_t fp8_slab_bytes;-} Slot;-typedef struct { Slot *slots; int *by_expert, n, cap; } LCache;-+#include "qwen36_tier.h" /* CUDA VRAM expert tier, shared with qwen36; CPU-only builds get the inline no-op stubs */+/* Native Qwen3.8-Flash-Next text core.+ *+ * This header is included once by qwen38.c after its tokenizer and protocol+ * helpers. It intentionally consumes the official HF safetensors layout:+ * the multimodal wrapper's `model.language_model` namespace and the standalone+ * text model's `model` namespace are both accepted. Vision and MTP tensors are+ * indexed by st.h but are never read.+ */+#ifndef COLI_QWEN38_CORE_H+#define COLI_QWEN38_CORE_H+#include <pthread.h> /* q38_ehit_mark publishes the lazy HITS table under a lock */++#define Q38_MAX_LAYERS 512+#define Q38_MAX_EXPERTS 1024+#define Q38_MAX_TOPK 256+#define Q38_MAX_PLE_PARTS 512+#define Q38_PREFILL_BATCH_ROWS 32+#define Q38_PREFILL_WORKSPACE_BYTES (64u << 20)++typedef struct {+ int hidden, layers, vocab, max_positions, eos_id;+ float eps, theta;+ int hc_count, hc_rank, hc_width;+ int q_heads, kv_heads, head_dim, rotary_dim;+ int idx_qheads, idx_kheads, idx_dim, idx_budget, idx_ratio;+ int experts, topk, inter, shared_inter, norm_topk;+ int dn_kheads, dn_vheads, dn_kdim, dn_vdim, dn_convk, dn_conv_dim;+ int ple_layer, ple_dim, ple_convk, ngram_size, heads_per_ngram;+ int ngram_heads, ngram_head_dim, ngram_parts;+ uint8_t *is_attn;+ /* vision: 0 = checkpoint di solo testo, o torre non caricata */+ int image_token;+ int vis_depth, vis_hidden, vis_heads, vis_inter, vis_patch;+ int vis_merge, vis_temporal, vis_in_ch, vis_out_hidden, vis_num_pos;+} Cfg;++typedef enum {+ Q38_WEIGHT_NONE = 0,+ Q38_WEIGHT_F32,+ Q38_WEIGHT_BF16,+ Q38_WEIGHT_FP8+} Q38WeightKind;++typedef struct {+ void *data;+ float *scales; /* block-FP8 only */+ int rows, cols;+ int64_t elements, scale_count;+ Q38WeightKind kind;+ unsigned owns_data:1, owns_scales:1;+ int gpu; /* 0 = CPU; else 1 + tier handle of an int8 copy resident in VRAM (decode, S == 1) */+ int8_t *q8; float *q8sc; /* Q38_TRUNK_CPU_INT8=1: the same int8 rows kept on the CPU (reference for the GPU path, no GPU needed) */+} Q38Weight;++typedef struct { float *norm; Q38Weight down, up, inject; } GatedResidual;++/* Persistent wall-clock counters. They live on Model rather than in process+ * globals so Segment sessions and future multi-model serving cannot leak phase+ * time into one another. Some categories intentionally overlap: architecture+ * phases (DeltaNet/QSA/PLE) contain resident matmuls, while the matmul counter+ * answers the orthogonal question "how much time is in dense kernels?". */+typedef enum {+ Q38_TM_EXPERT_READ = 0,+ Q38_TM_FP8_EXPAND,+ Q38_TM_ROUTED_EXPERT,+ Q38_TM_SHARED_EXPERT,+ Q38_TM_DENSE_MATMUL,+ Q38_TM_DELTANET,+ Q38_TM_QSA_INDEX,+ Q38_TM_QSA_ATTENTION,+ Q38_TM_PLE,+ Q38_TM_LM_HEAD,+ Q38_TM_COUNT+} Q38Timer;++typedef struct {+ double seconds[Q38_TM_COUNT];+ uint64_t forwards;+} Q38Timers;++typedef struct {+ GatedResidual attn_gr, mlp_gr;+ Q38Weight router, sh_g, sh_u, sh_d;+ float *sh_gate;+ Q38Weight q, k, v, o;+ float *qn, *kn;+ Q38Weight idx_qk;+ float *idx_qn, *idx_kn;+ Q38Weight dn_qkv, dn_z, dn_b, dn_a, dn_out;+ float *dn_conv;+ float *dn_dtbias, *dn_alog, *dn_norm;+ Q38Weight ple_key, ple_value;+ float *ple_norm_key, *ple_norm_query;+ float *ple_norm_conv, *ple_conv;+} Layer;++typedef struct {+ int eid;+ Q38Weight gate, up, down;+ uint64_t used;+ void *fp8_slab;+ int64_t fp8_slab_bytes;+} Slot;+typedef struct { Slot *slots; int *by_expert, n, cap; } LCache;+typedef struct {float *values; /* [expert][gate,up,down][block] */int64_t scale_count;@@ -119,1197 +119,1197 @@ typedef enum {Q38_EXPERT_BATCH_FALLBACK_DUPLICATE,Q38_EXPERT_BATCH_FALLBACK_LAYOUT,} Q38ExpertBatchFallback;--typedef struct {- Cfg c;- shards S;- char prefix[32];- Q38Weight embed, lm_head;- GatedResidual final_gr;- Layer *L;- LCache *cache;- uint8_t **ehit; /* experts routed this turn, for HITS (dashboard Brain) */- Q38ExpertScaleCache *expert_scales;- uint64_t clock, hits, miss;- uint64_t expert_weight_reads, expert_scale_reads, expert_pair_reads;- uint64_t expert_prefetch_ranges, expert_parallel_batches;- uint64_t expert_scale_bytes;- float **DN_rec, **DN_conv;- float **K, **V, **IK;- int kv_len, kv_cap, max_t;- st_tensor *ple_parts[Q38_MAX_PLE_PARTS];- char ple_part_names[Q38_MAX_PLE_PARTS][320];- int64_t ple_part_start[Q38_MAX_PLE_PARTS + 1];- int ple_part_count;- float ple_weight_scale;- int64_t ple_multipliers[3], ple_head_vocab[64], ple_head_offset[64];- int64_t *ple_history;- float *PLE_conv_state;- int ple_history_len;++typedef struct {+ Cfg c;+ shards S;+ char prefix[32];+ Q38Weight embed, lm_head;+ GatedResidual final_gr;+ Layer *L;+ LCache *cache;+ uint8_t **ehit; /* experts routed this turn, for HITS (dashboard Brain) */+ Q38ExpertScaleCache *expert_scales;+ uint64_t clock, hits, miss;+ uint64_t expert_weight_reads, expert_scale_reads, expert_pair_reads;+ uint64_t expert_prefetch_ranges, expert_parallel_batches;+ uint64_t expert_scale_bytes;+ float **DN_rec, **DN_conv;+ float **K, **V, **IK;+ int kv_len, kv_cap, max_t;+ st_tensor *ple_parts[Q38_MAX_PLE_PARTS];+ char ple_part_names[Q38_MAX_PLE_PARTS][320];+ int64_t ple_part_start[Q38_MAX_PLE_PARTS + 1];+ int ple_part_count;+ float ple_weight_scale;+ int64_t ple_multipliers[3], ple_head_vocab[64], ple_head_offset[64];+ int64_t *ple_history;+ float *PLE_conv_state;+ int ple_history_len;int range_begin, range_end;int native_fp8, native_bf16, expert_prefetch, expert_parallel_reads;Q38ExpertBatchFallback expert_batch_fallback;int prefill_batch;- uint64_t resident_weight_bytes;- double dense_load_s;- /* vision. `vis_map` mappa la posizione ASSOLUTA nella sequenza alla riga di- * `vis_rows`, oppure -1. Assoluta e non relativa al chunk: il prefill arriva- * a pezzi, e un indice relativo darebbe l'immagine sbagliata al secondo- * pezzo senza che niente protesti. */- /* PLE prefetto: le righe gia' lette per il chunk in corso, o NULL. */- float *ple_pref; int ple_pref_rows;- Q38Vision vis;- int vis_ready;- float *vis_rows;- int *vis_map, vis_map_len, vis_rows_n;- Q38Timers timers;-} Model;--static float *g_last_logit;-static int g_capture_last_logit;--static double now_s(void) {- struct timespec t;- clock_gettime(CLOCK_MONOTONIC, &t);- return (double)t.tv_sec + (double)t.tv_nsec * 1e-9;-}--static inline void q38_tm_add(Model *m,Q38Timer timer,double started) {- m->timers.seconds[timer] += now_s() - started;-}--static Q38Timers q38_tm_delta(const Q38Timers *after,const Q38Timers *before) {- Q38Timers delta={0};- for(int i=0;i<Q38_TM_COUNT;i++)- delta.seconds[i]=after->seconds[i]-before->seconds[i];- delta.forwards=after->forwards-before->forwards;- return delta;-}--#if defined(__APPLE__)-static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF,&r); return r.ru_maxrss/1073741824.0; }-#else-static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF,&r); return r.ru_maxrss/1048576.0; }-#endif--static float *falloc(int64_t n) {- if (n < 0 || (uint64_t)n > SIZE_MAX / sizeof(float)) {- fprintf(stderr, "invalid float allocation: %lld\n", (long long)n); exit(1);- }- float *p = (float *)malloc((size_t)n * sizeof(float));- if (!p && n) { fprintf(stderr, "OOM allocating %lld floats\n", (long long)n); exit(1); }- return p;-}--/* W is row-major [O,I], y=x@W^T. */-static void q38_matmul(float *y, const float *x, const float *W, int S, int I, int O) {- #pragma omp parallel for schedule(static)- for (int o = 0; o < O; o++) {- const float *w = W + (int64_t)o * I;- for (int s = 0; s < S; s++) {- const float *xs = x + (int64_t)s * I;- float a = 0.f;- for (int i = 0; i < I; i++) a += xs[i] * w[i];- y[(int64_t)s * O + o] = a;- }- }-}--static void q38_weight_free(Q38Weight *weight) {- if(!weight)return;- if(weight->owns_data)free(weight->data);- if(weight->owns_scales)free(weight->scales);- free(weight->q8); free(weight->q8sc);- memset(weight,0,sizeof(*weight));-}--static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,int cols) {- if(rows<=0||cols<=0||(int64_t)rows>INT64_MAX/cols){- fprintf(stderr,"invalid weight geometry [%d,%d]\n",rows,cols);exit(1);- }- int64_t elements=(int64_t)rows*cols;- int64_t scales=kind==Q38_WEIGHT_FP8?fp8_nblk(rows)*fp8_nblk(cols):0;- if(weight->kind==kind&&weight->rows==rows&&weight->cols==cols&&- weight->owns_data&&weight->data&&- (!scales||(weight->owns_scales&&weight->scales)))return;- q38_weight_free(weight);- size_t element_size=kind==Q38_WEIGHT_F32?sizeof(float):- kind==Q38_WEIGHT_BF16?sizeof(uint16_t):- kind==Q38_WEIGHT_FP8?sizeof(uint8_t):0;- if(!element_size||(uint64_t)elements>SIZE_MAX/element_size||- (scales&&(uint64_t)scales>SIZE_MAX/sizeof(float))){- fprintf(stderr,"unsupported or oversized weight geometry [%d,%d] kind=%d\n",- rows,cols,(int)kind);exit(1);- }- weight->data=malloc((size_t)elements*element_size);- if(scales)weight->scales=(float*)malloc((size_t)scales*sizeof(float));- if(!weight->data||(scales&&!weight->scales)){- fprintf(stderr,"OOM allocating weight [%d,%d] kind=%d\n",rows,cols,(int)kind);exit(1);- }- weight->rows=rows;weight->cols=cols;weight->elements=elements;- weight->scale_count=scales;weight->kind=kind;- weight->owns_data=1;weight->owns_scales=scales?1u:0u;-}--static uint64_t q38_weight_bytes(const Q38Weight *weight) {- uint64_t element_size=weight->kind==Q38_WEIGHT_F32?sizeof(float):- weight->kind==Q38_WEIGHT_BF16?sizeof(uint16_t):- weight->kind==Q38_WEIGHT_FP8?sizeof(uint8_t):0;- return (uint64_t)weight->elements*element_size+- (uint64_t)weight->scale_count*sizeof(float);-}--/* Native BF16 storage with FP32 activations and accumulation. This deliberately- * does not round activations to BF16 or use BF16 dot-product instructions: it is- * the storage-equivalent form of the existing st_read_f32 reference. */-static void q38_matmul_bf16(float *y,const float *x,const uint16_t *W,- int S,int I,int O) {- #pragma omp parallel for schedule(static)- for(int o=0;o<O;o++){- const uint16_t *w=W+(int64_t)o*I;- for(int s=0;s<S;s++){- const float *xs=x+(int64_t)s*I;float a=0.f;- for(int i=0;i<I;i++)a+=xs[i]*bf16_to_f32(w[i]);- y[(int64_t)s*O+o]=a;- }- }-}--static void q38_weight_matmul(float *y,const float *x,const Q38Weight *weight,- int S,int I,int O) {- /* A matrix the tier placed in VRAM (q38_trunk_place) answers a decode- * GEMV from there; prefill rows and any failure take the CPU path below,- * so the BF16 copy stays the reference for everything but S == 1. */- if(S==1&&weight&&weight->gpu&&weight->rows==O&&weight->cols==I&&- qt_dense_matmul(weight->gpu-1,y,x,I,O))return;- if(S==1&&weight&&weight->q8&&weight->rows==O&&weight->cols==I){- /* the int8 rows the GPU would hold, computed here: what the trunk- * quantization alone does to the output, GPU or not */- const int8_t *q=weight->q8; const float *sc=weight->q8sc;- #pragma omp parallel for schedule(static)- for(int o=0;o<O;o++){- const int8_t *w=q+(size_t)o*I; float a=0.f;- for(int i=0;i<I;i++)a+=x[i]*(float)w[i];- y[o]=a*sc[o];- }- return;- }- if(!weight||weight->rows!=O||weight->cols!=I||!weight->data){- fprintf(stderr,"invalid matmul weight: have [%d,%d] kind=%d, need [%d,%d]\n",- weight?weight->rows:0,weight?weight->cols:0,- weight?(int)weight->kind:0,O,I);exit(1);- }- if(weight->kind==Q38_WEIGHT_F32)- q38_matmul(y,x,(const float*)weight->data,S,I,O);- else if(weight->kind==Q38_WEIGHT_BF16)- q38_matmul_bf16(y,x,(const uint16_t*)weight->data,S,I,O);- else if(weight->kind==Q38_WEIGHT_FP8&&weight->scales)- matmul_fp8(y,x,(const uint8_t*)weight->data,weight->scales,S,I,O);- else {fprintf(stderr,"unsupported matmul weight kind %d\n",(int)weight->kind);exit(1);}-}--static void q38_weight_row(const Q38Weight *weight,int row,float *out) {- if(!weight||row<0||row>=weight->rows||!weight->data){- fprintf(stderr,"invalid weight row %d\n",row);exit(1);- }- if(weight->kind==Q38_WEIGHT_F32)- memcpy(out,(const float*)weight->data+(int64_t)row*weight->cols,- (size_t)weight->cols*sizeof(float));- else if(weight->kind==Q38_WEIGHT_BF16){- const uint16_t *src=(const uint16_t*)weight->data+(int64_t)row*weight->cols;- for(int i=0;i<weight->cols;i++)out[i]=bf16_to_f32(src[i]);- } else {fprintf(stderr,"weight kind %d has no dense row view\n",(int)weight->kind);exit(1);}-}--static void q38_dense_matmul(Model *m,float *y,const float *x,const Q38Weight *weight,- int S,int I,int O) {- double started=now_s();- q38_weight_matmul(y,x,weight,S,I,O);- q38_tm_add(m,Q38_TM_DENSE_MATMUL,started);-}--static inline float q38_sigmoid(float x) {- if (x >= 0.f) { float z=expf(-x); return 1.f/(1.f+z); }- float z=expf(x); return z/(1.f+z);-}-static inline float q38_silu(float x) { return x * q38_sigmoid(x); }-static inline float q38_softplus(float x) { return x > 20.f ? x : log1pf(expf(x)); }--/* Qwen4-Exp RMSNorms are zero-centered: the learned scale is 1+weight. */-static void q38_rms0(float *out, const float *x, const float *w, int n, float eps) {- double ss=0.0; for (int i=0;i<n;i++) ss+=(double)x[i]*x[i];- float r=1.f/sqrtf((float)(ss/n)+eps);- for (int i=0;i<n;i++) out[i]=x[i]*r*(1.f+w[i]);-}--/* DeltaNet's RMSNormGated is inherited from Qwen3-Next and is not zero-centered. */-static void q38_rmsg(float *out,const float *x,const float *gate,const float *w,- int n,float eps,int sigmoid_gate) {- double ss=0.0; for(int i=0;i<n;i++) ss+=(double)x[i]*x[i];- float r=1.f/sqrtf((float)(ss/n)+eps);- for(int i=0;i<n;i++) out[i]=x[i]*r*w[i]*(sigmoid_gate?q38_sigmoid(gate[i]):q38_silu(gate[i]));-}--static void q38_rope(float *x, int dim, int rotary_dim, int pos, float theta) {- int half=rotary_dim/2;- for(int i=0;i<half;i++) {- float ang=(float)pos/powf(theta,(float)(2*i)/rotary_dim);- float co=cosf(ang), si=sinf(ang), a=x[i], b=x[i+half];- x[i]=a*co-b*si; x[i+half]=b*co+a*si;- }- (void)dim;-}--static jval *q38_obj(jval *o,const char *key) {- jval *v=json_get(o,key);- if(v&&v->t!=J_OBJ){fprintf(stderr,"config.json: %s must be an object\n",key);exit(1);}- return v;-}-static double q38_num(jval *o,const char *key,double def,int required) {- jval *v=json_get(o,key);- if(v&&v->t==J_NUM) return v->num;- if(v){fprintf(stderr,"config.json: %s must be numeric\n",key);exit(1);}- if(required){fprintf(stderr,"config.json: missing numeric %s\n",key);exit(1);} return def;-}-/* JSON numbers are doubles in the small parser. Never narrow an unchecked- * value to an int: besides accepting fractional dimensions, a huge finite- * value can become implementation-defined before q38_validate_cfg sees it. */-static int q38_num_int(jval *o,const char *key,double def,int required,- int min_value,int max_value) {- double value=q38_num(o,key,def,required);- if(!isfinite(value)||floor(value)!=value||value<(double)min_value||- value>(double)max_value){- fprintf(stderr,"config.json: %s must be an integer in [%d,%d]\n",- key,min_value,max_value);exit(1);- }- return (int)value;-}-static int q38_derived_product(const char *name,int left,int right) {- if(left<0||right<0||(uint64_t)left*(uint64_t)right>(uint64_t)INT_MAX){- fprintf(stderr,"[qwen38 config] derived %s overflows int\n",name);exit(1);- }- return left*right;-}-static int q38_derived_sum(const char *name,int left,int right) {- if(left<0||right<0||left>INT_MAX-right){- fprintf(stderr,"[qwen38 config] derived %s overflows int\n",name);exit(1);- }- return left+right;-}-static int q38_bool(jval *o,const char *key,int def) {- jval *v=json_get(o,key);- if(v&&v->t!=J_BOOL){fprintf(stderr,"config.json: %s must be boolean\n",key);exit(1);}- return v?v->boolean:def;-}-static const char *q38_string(jval *o,const char *key,const char *def) {- jval *v=json_get(o,key);- if(v&&v->t!=J_STR){fprintf(stderr,"config.json: %s must be a string\n",key);exit(1);}- return v?v->str:def;-}-static void q38_require_string(jval *o,const char *key,const char *expected) {- const char *value=q38_string(o,key,expected);- if(strcmp(value,expected)){- fprintf(stderr,"config.json: unsupported %s=%s (expected %s)\n",- key,value,expected);exit(1);- }-}-static void q38_require_present_string(jval *o,const char *key,- const char *expected) {- jval *value=json_get(o,key);- if(!value||value->t!=J_STR||strcmp(value->str,expected)){- fprintf(stderr,"config.json: %s must be explicitly set to %s\n",- key,expected);exit(1);- }-}-static void q38_require_bool(jval *o,const char *key,int expected) {- int value=q38_bool(o,key,expected);- if(value!=expected){- fprintf(stderr,"config.json: unsupported %s=%s (expected %s)\n",key,- value?"true":"false",expected?"true":"false");exit(1);- }-}--static void q38_load_cfg(Cfg *c,const char *snap) {- memset(c,0,sizeof(*c));- char path[2048]; snprintf(path,sizeof path,"%s/config.json",snap);- FILE *f=fopen(path,"rb"); if(!f){perror(path);exit(1);} fseek(f,0,SEEK_END);- long n=ftell(f); fseek(f,0,SEEK_SET);- if(n<0||n>(256L<<20)){fprintf(stderr,"invalid config size\n");exit(1);}- char *buf=(char*)malloc((size_t)n+1),*arena=NULL;- if(!buf||fread(buf,1,(size_t)n,f)!=(size_t)n){fprintf(stderr,"cannot read %s\n",path);exit(1);}- fclose(f); buf[n]=0; jval *root=json_parse(buf,&arena);- if(!root||root->t!=J_OBJ){fprintf(stderr,"invalid %s\n",path);exit(1);}- jval *tc=q38_obj(root,"text_config"); if(!tc) tc=root;- const char *mt=jstr(tc,"model_type");- if(!mt||strcmp(mt,"qwen4_exp_text")){fprintf(stderr,"unsupported text model_type: %s\n",mt?mt:"(missing)");exit(1);}- q38_require_string(tc,"hidden_act","silu");- /* Upstream's omitted/None default resolves to hidden_act (SiLU), whereas- * this implementation deliberately uses the released sigmoid gate. */- q38_require_present_string(tc,"output_gate_type","sigmoid");- q38_require_bool(tc,"attention_bias",0);- q38_require_bool(tc,"tie_word_embeddings",0);- c->hidden=q38_num_int(tc,"hidden_size",0,1,0,INT_MAX);- c->layers=q38_num_int(tc,"num_hidden_layers",0,1,0,Q38_MAX_LAYERS);- c->vocab=q38_num_int(tc,"vocab_size",0,1,0,INT_MAX);- c->max_positions=q38_num_int(tc,"max_position_embeddings",0,1,0,INT_MAX);- c->eos_id=q38_num_int(tc,"eos_token_id",-1,1,INT_MIN,INT_MAX);- double eps=q38_num(tc,"rms_norm_eps",1e-6,0);- if(!isfinite(eps)||eps<=0.0||eps>FLT_MAX){fprintf(stderr,"invalid rms_norm_eps\n");exit(1);}- c->eps=(float)eps;- jval *rp=q38_obj(tc,"rope_parameters");- if(rp)q38_require_string(rp,"rope_type","default");- double theta=rp?q38_num(rp,"rope_theta",10000,0):q38_num(tc,"rope_theta",10000,0);- if(!isfinite(theta)||theta<=0.0||theta>FLT_MAX){fprintf(stderr,"invalid rope_theta\n");exit(1);}- c->theta=(float)theta;- c->hc_count=q38_num_int(tc,"hc_count",4,0,0,INT_MAX);- c->hc_rank=q38_num_int(tc,"hc_lowrank",320,0,0,INT_MAX);- c->hc_width=q38_derived_product("hc_width",c->hc_count,c->hidden);- c->q_heads=q38_num_int(tc,"num_attention_heads",0,1,0,INT_MAX);- c->kv_heads=q38_num_int(tc,"num_key_value_heads",0,1,0,INT_MAX);- c->head_dim=q38_num_int(tc,"head_dim",0,1,0,INT_MAX);- double partial=rp?q38_num(rp,"partial_rotary_factor",q38_num(tc,"partial_rotary_factor",1,0),0):q38_num(tc,"partial_rotary_factor",1,0);- double rotary_product=(double)c->head_dim*partial;- if(!isfinite(partial)||!isfinite(rotary_product)||rotary_product<0||- rotary_product>(double)INT_MAX){- fprintf(stderr,"[qwen38 config] derived rotary_dim overflows int\n");exit(1);- }- /* Upstream Qwen4ExpTextConfig derives this with Python int(), i.e. truncation- * toward zero rather than rounding to the nearest dimension. */- c->rotary_dim=(int)rotary_product;- c->idx_qheads=q38_num_int(tc,"indexer_n_heads",0,1,0,INT_MAX);- c->idx_kheads=q38_num_int(tc,"indexer_kv_heads",0,1,0,INT_MAX);- c->idx_dim=q38_num_int(tc,"indexer_head_dim",0,1,0,INT_MAX);- c->idx_budget=q38_num_int(tc,"indexer_budget",0,1,0,INT_MAX);- c->idx_ratio=q38_num_int(tc,"indexer_compress_ratio",0,1,0,INT_MAX);- c->experts=q38_num_int(tc,"num_experts",0,1,0,INT_MAX);- c->topk=q38_num_int(tc,"num_experts_per_tok",0,1,0,INT_MAX);- c->inter=q38_num_int(tc,"moe_intermediate_size",0,1,0,INT_MAX);- c->shared_inter=q38_num_int(tc,"shared_expert_intermediate_size",0,1,0,INT_MAX);- c->norm_topk=q38_bool(tc,"norm_topk_prob",1);- c->dn_kheads=q38_num_int(tc,"linear_num_key_heads",0,1,0,INT_MAX);- c->dn_vheads=q38_num_int(tc,"linear_num_value_heads",0,1,0,INT_MAX);- c->dn_kdim=q38_num_int(tc,"linear_key_head_dim",0,1,0,INT_MAX);- c->dn_vdim=q38_num_int(tc,"linear_value_head_dim",0,1,0,INT_MAX);- c->dn_convk=q38_num_int(tc,"linear_conv_kernel_dim",0,1,0,INT_MAX);- int dn_qk=q38_derived_product("deltanet_qk",c->dn_kheads,c->dn_kdim);- dn_qk=q38_derived_product("deltanet_qk_twice",2,dn_qk);- c->dn_conv_dim=q38_derived_sum("dn_conv_dim",dn_qk,- q38_derived_product("deltanet_v",c->dn_vheads,c->dn_vdim));- c->ple_dim=q38_num_int(tc,"ple_embed_dim",c->hidden,0,0,INT_MAX);- c->ple_convk=q38_num_int(tc,"ple_conv_kernel_size",4,0,0,INT_MAX);- c->ngram_size=q38_num_int(tc,"ngram_size",3,0,0,INT_MAX);- c->heads_per_ngram=q38_num_int(tc,"heads_per_ngram",8,0,0,INT_MAX);- c->ngram_heads=(c->ngram_size>0)?q38_derived_product("ngram_heads",c->ngram_size-1,c->heads_per_ngram):0;- if(c->ngram_heads<=0){fprintf(stderr,"[qwen38 config] derived ngram_heads invalid\n");exit(1);}- c->ngram_head_dim=c->ple_dim/c->ngram_heads;- c->ngram_parts=q38_num_int(tc,"split_ngram_parts",1,0,0,INT_MAX);- c->ple_layer=-1;- jval *pl=json_get(tc,"ple_layer_ids");- if(pl&&pl->t==J_ARR&&pl->len){- if(pl->len!=1||pl->kids[0]->t!=J_NUM){fprintf(stderr,"only one PLE layer is supported\n");exit(1);}- double raw=pl->kids[0]->num;- if(!isfinite(raw)||floor(raw)!=raw||raw<1||raw>(double)INT_MAX){- fprintf(stderr,"config.json: ple_layer_ids[0] must be a positive integer\n");exit(1);- }- int one_based=(int)raw;- c->ple_layer=one_based-1;- }- c->is_attn=(uint8_t*)calloc((size_t)c->layers,1);- jval *lt=json_get(tc,"layer_types");- if(!lt||lt->t!=J_ARR||lt->len!=c->layers){fprintf(stderr,"config layer_types must have %d entries\n",c->layers);exit(1);}- for(int i=0;i<c->layers;i++){- const char *s=lt->kids[i]->t==J_STR?lt->kids[i]->str:NULL;- if(!s){fprintf(stderr,"invalid layer_types[%d]\n",i);exit(1);}- if(!strcmp(s,"linear_attention")) c->is_attn[i]=0;- else if(!strcmp(s,"full_attention")||!strcmp(s,"qwen_sparse_attention")) c->is_attn[i]=1;- else {fprintf(stderr,"unsupported layer type %s\n",s);exit(1);}- }- /* Vision: opzionale. Un checkpoint di solo testo non ha vision_config, e in- * quel caso la torre resta spenta invece di rifiutare il modello. La- * geometria viene dal file, non da costanti qui: una torre di misura diversa- * deve fallire dicendo cosa non torna, non leggere pesi della misura- * sbagliata. */- {- jval *vc = q38_obj(root, "vision_config");- if (vc) {- c->vis_depth = q38_num_int(vc,"depth",0,1,1,1024);- c->vis_hidden = q38_num_int(vc,"hidden_size",0,1,1,65536);- c->vis_heads = q38_num_int(vc,"num_heads",0,1,1,1024);- c->vis_inter = q38_num_int(vc,"intermediate_size",0,1,1,262144);- c->vis_patch = q38_num_int(vc,"patch_size",16,0,1,512);- c->vis_merge = q38_num_int(vc,"spatial_merge_size",2,0,1,8);- c->vis_temporal = q38_num_int(vc,"temporal_patch_size",2,0,1,8);- c->vis_in_ch = q38_num_int(vc,"in_channels",3,0,1,8);- c->vis_out_hidden = q38_num_int(vc,"out_hidden_size",c->hidden,0,1,65536);- c->vis_num_pos = q38_num_int(vc,"num_position_embeddings",0,1,1,1<<20);- c->image_token = q38_num_int(root,"image_token_id",-1,0,0,INT_MAX);- }- }- json_free(root); free(buf); free(arena);-}--#define Q38_NEED(x,...) do{if(!(x)){fprintf(stderr,"[qwen38 config] ");fprintf(stderr,__VA_ARGS__);fprintf(stderr," -- refusing\n");exit(1);}}while(0)-static void q38_validate_cfg(const Cfg *c) {- Q38_NEED(c->hidden>0&&c->hidden<=65536,"hidden_size=%d",c->hidden);- if (c->vis_depth) {- /* La torre proietta direttamente nello spazio del testo: se le due- * dimensioni non coincidono i token immagine finirebbero nel posto- * giusto con i numeri sbagliati, che e' peggio di un rifiuto. */- Q38_NEED(c->vis_out_hidden==c->hidden,- "vision out_hidden_size=%d but text hidden_size=%d",- c->vis_out_hidden,c->hidden);- Q38_NEED(c->vis_hidden%c->vis_heads==0,- "vision hidden_size=%d not divisible by num_heads=%d",- c->vis_hidden,c->vis_heads);- Q38_NEED(c->image_token>=0&&c->image_token<c->vocab,- "image_token_id=%d outside vocabulary",c->image_token);- }- Q38_NEED(c->layers>0&&c->layers<=Q38_MAX_LAYERS,"layers=%d",c->layers);- Q38_NEED(c->vocab>0&&c->max_positions>0&&- c->max_positions<=QWEN38_ATTN_MAX_CTX,"vocab/context invalid");- Q38_NEED(isfinite(c->eps)&&c->eps>0.f&&isfinite(c->theta)&&c->theta>0.f,"RoPE/norm constants invalid");- Q38_NEED(c->eos_id>=0&&c->eos_id<c->vocab,"eos=%d",c->eos_id);- Q38_NEED(c->hc_count>1&&c->hc_count<=16&&c->hc_rank>0&&- c->hc_rank<=65536&&c->hc_width>0,"gated residual dimensions invalid");- Q38_NEED(c->q_heads>0&&c->kv_heads>0&&c->q_heads%c->kv_heads==0,"attention heads invalid");- Q38_NEED(c->head_dim>0&&c->rotary_dim>0&&!(c->rotary_dim&1)&&c->rotary_dim<=c->head_dim,"RoPE dimensions invalid");- Q38_NEED(c->head_dim<=INT_MAX/2&&c->q_heads<=INT_MAX/(2*c->head_dim)&&- c->kv_heads<=INT_MAX/c->head_dim,"attention projection dimensions overflow");- Q38_NEED(c->idx_qheads>0&&c->idx_kheads==1&&c->idx_dim>=c->rotary_dim,"indexer dimensions invalid");- Q38_NEED(c->idx_ratio>0&&c->idx_budget>0&&c->idx_budget%c->idx_ratio==0&&- c->idx_budget<=INT_MAX-c->idx_ratio+1,"indexer budget invalid");- Q38_NEED(c->idx_qheads<INT_MAX&&c->idx_qheads<=INT_MAX/c->idx_dim-c->idx_kheads,- "indexer projection dimensions overflow");- Q38_NEED(c->experts>0&&c->experts<=Q38_MAX_EXPERTS,"experts=%d",c->experts);- Q38_NEED(c->topk>0&&c->topk<=Q38_MAX_TOPK&&c->topk<=c->experts,"topk=%d",c->topk);- Q38_NEED(c->inter<=INT_MAX/2,"expert projection dimensions overflow");- Q38_NEED(c->inter>0&&c->shared_inter>0,"MoE widths invalid");- Q38_NEED(c->dn_vheads>0&&c->dn_kheads>0&&c->dn_vheads%c->dn_kheads==0,"DeltaNet heads invalid");- Q38_NEED(c->dn_kdim>0&&c->dn_vdim>0&&c->dn_vdim<=512&&c->dn_convk>=2&&c->dn_conv_dim>0&&- c->dn_vdim<=INT_MAX/c->dn_vheads&&c->dn_kdim<=INT_MAX/c->dn_vdim,- "DeltaNet dimensions invalid");- Q38_NEED((uint64_t)c->dn_vheads<=SIZE_MAX/sizeof(float)/(uint64_t)c->dn_kdim/(uint64_t)c->dn_vdim&&- (uint64_t)c->dn_conv_dim<=SIZE_MAX/sizeof(float)/(uint64_t)(c->dn_convk-1),- "DeltaNet state dimensions overflow");- Q38_NEED(c->ngram_size==3&&c->ngram_heads>0&&c->ngram_heads<=64&&- c->ple_dim>0&&c->ple_dim%c->ngram_heads==0&&- c->ngram_head_dim>0&&c->ngram_head_dim<=512&&- c->ple_convk>=2&&c->ple_convk<=INT_MAX/c->ngram_size,- "PLE dimensions invalid");- Q38_NEED((uint64_t)c->hc_width<=SIZE_MAX/sizeof(float)/(uint64_t)(c->ple_convk-1)/(uint64_t)c->ngram_size,- "PLE state dimensions overflow");- Q38_NEED(c->ngram_parts>0&&c->ngram_parts<=Q38_MAX_PLE_PARTS,"PLE parts=%d",c->ngram_parts);- Q38_NEED(c->ple_layer>=0&&c->ple_layer<c->layers,"PLE layer=%d",c->ple_layer);-}--static float *q38_load_tensor(Model *m,const char *name,int64_t want) {- st_tensor *t=st_find(&m->S,name);- if(!t){fprintf(stderr,"missing %s\n",name);exit(1);}- if(t->numel!=want){fprintf(stderr,"%s: %lld elements, expected %lld\n",name,(long long)t->numel,(long long)want);exit(1);}- float *p=falloc(want); st_read_f32(&m->S,name,p,1); return p;-}--static int q38_env_bool(const char *name,int default_value) {- const char *value=getenv(name);if(!value||!*value)return default_value;- if(value[0]=='0'&&!value[1])return 0;- if(value[0]=='1'&&!value[1])return 1;- fprintf(stderr,"%s must be exactly 0 or 1\n",name);exit(1);-}--static Q38Weight q38_load_weight(Model *m,const char *name,int rows,int cols) {- st_tensor *tensor=st_find(&m->S,name);Q38Weight weight={0};- if(!tensor){fprintf(stderr,"missing %s\n",name);exit(1);}- if(tensor->rank!=2||tensor->shape[0]!=rows||tensor->shape[1]!=cols||- tensor->numel!=(int64_t)rows*cols){- fprintf(stderr,"%s: invalid matrix shape, expected [%d,%d]\n",name,rows,cols);exit(1);- }- if(tensor->dtype==0&&m->native_bf16){- q38_weight_reserve(&weight,Q38_WEIGHT_BF16,rows,cols);- if(tensor->nbytes!=weight.elements*(int64_t)sizeof(uint16_t)){- fprintf(stderr,"%s: invalid BF16 byte count\n",name);exit(1);- }- st_read_raw_cap(&m->S,name,weight.data,tensor->nbytes,1);- } else {- if(tensor->dtype<0||tensor->dtype>2){- fprintf(stderr,"%s: unsupported resident dtype %s\n",name,st_dtype_name(tensor->dtype));exit(1);- }- q38_weight_reserve(&weight,Q38_WEIGHT_F32,rows,cols);- st_read_f32(&m->S,name,(float*)weight.data,1);- }- m->resident_weight_bytes+=q38_weight_bytes(&weight);- return weight;-}--static void q38_name(Model *m,char *out,size_t cap,int layer,const char *suffix) {- snprintf(out,cap,"%s.layers.%d.%s",m->prefix,layer,suffix);-}--static void q38_load_gr(Model *m,GatedResidual *g,int layer,const char *kind,int inject) {- Cfg *c=&m->c; char nm[320],base[180];- if(layer>=0) snprintf(base,sizeof base,"layers.%d.%s",layer,kind); else snprintf(base,sizeof base,"hyper_connection_mixer");- snprintf(nm,sizeof nm,"%s.%s.hc_norm.weight",m->prefix,base); g->norm=q38_load_tensor(m,nm,c->hc_width);- snprintf(nm,sizeof nm,"%s.%s.input_mix_weight_down.weight",m->prefix,base); g->down=q38_load_weight(m,nm,c->hc_rank,c->hc_width);- snprintf(nm,sizeof nm,"%s.%s.input_mix_weight_up.weight",m->prefix,base); g->up=q38_load_weight(m,nm,c->hc_width,c->hc_rank);- if(inject){snprintf(nm,sizeof nm,"%s.%s.block_inject_weight.weight",m->prefix,base);g->inject=q38_load_weight(m,nm,c->hc_count,c->hc_width);}-}--static void q38_load_ple(Model *m,Layer *l) {- Cfg *c=&m->c; int i=c->ple_layer; char nm[320];- q38_name(m,nm,sizeof nm,i,"ple.key_proj.weight");l->ple_key=q38_load_weight(m,nm,c->hc_width,c->ple_dim);- q38_name(m,nm,sizeof nm,i,"ple.value_proj.weight");l->ple_value=q38_load_weight(m,nm,c->hidden,c->ple_dim);- #define PL(field,suf,n) q38_name(m,nm,sizeof nm,i,"ple." suf); l->field=q38_load_tensor(m,nm,(n))- PL(ple_norm_key,"norm_key.weight",c->hc_width);PL(ple_norm_query,"norm_query.weight",c->hc_width);- PL(ple_norm_conv,"norm_conv.weight",c->hc_width);PL(ple_conv,"conv1d.weight",(int64_t)c->hc_width*c->ple_convk);- #undef PL- const char *bufs[]={"layer_multipliers","ngram_heads_vocab_sizes","ngram_heads_offsets"};- int64_t *dsts[]={m->ple_multipliers,m->ple_head_vocab,m->ple_head_offset};- int counts[]={c->ngram_size,c->ngram_heads,c->ngram_heads};- for(int b=0;b<3;b++){- q38_name(m,nm,sizeof nm,i,"ple.ple_embedding."); strncat(nm,bufs[b],sizeof(nm)-strlen(nm)-1);- st_tensor *t=st_find(&m->S,nm); if(!t||t->dtype!=6||t->numel!=counts[b]){fprintf(stderr,"invalid %s\n",nm);exit(1);}- st_read_raw_cap(&m->S,nm,dsts[b],(int64_t)counts[b]*8,1);- }- for(int h=0;h<c->ngram_heads;h++)- Q38_NEED(m->ple_head_vocab[h]>0&&m->ple_head_offset[h]>=0&&- m->ple_head_vocab[h]<=INT64_MAX-m->ple_head_offset[h],- "invalid PLE head range %d",h);- m->ple_part_count=0; m->ple_part_start[0]=0;- q38_name(m,nm,sizeof nm,i,"ple.ple_embedding.ngram_embedding.weight");- if(st_has(&m->S,nm)){- st_tensor *t=st_find(&m->S,nm);- if(!t||t->rank!=2||t->shape[0]<=0||t->shape[1]!=c->ngram_head_dim){fprintf(stderr,"invalid PLE table %s\n",nm);exit(1);}- snprintf(m->ple_part_names[0],sizeof m->ple_part_names[0],"%s",nm);- m->ple_parts[0]=t; m->ple_part_count=1;- m->ple_part_start[1]=m->ple_parts[0]->shape[0];- } else {- for(int p=0;p<c->ngram_parts;p++){- q38_name(m,nm,sizeof nm,i,"ple.ple_embedding.ngram_embedding.");- size_t z=strlen(nm); snprintf(nm+z,sizeof(nm)-z,"shard_%d.weight",p);- st_tensor *t=st_find(&m->S,nm); if(!t){fprintf(stderr,"missing PLE shard %s\n",nm);exit(1);}- if(t->rank!=2||t->shape[0]<=0||t->shape[1]!=c->ngram_head_dim||- m->ple_part_start[p]>INT64_MAX-t->shape[0]){fprintf(stderr,"invalid PLE shard %s\n",nm);exit(1);}- snprintf(m->ple_part_names[p],sizeof m->ple_part_names[p],"%s",nm);- m->ple_parts[p]=t; m->ple_part_start[p+1]=m->ple_part_start[p]+t->shape[0];- m->ple_part_count++;- }- }- q38_name(m,nm,sizeof nm,i,"ple.ple_embedding.ngram_embedding.weight_scale");- m->ple_weight_scale=1.f;- if(st_has(&m->S,nm)){- st_tensor *t=st_find(&m->S,nm); if(t->numel!=1){fprintf(stderr,"invalid %s\n",nm);exit(1);}- st_read_f32(&m->S,nm,&m->ple_weight_scale,1);- }- int64_t need=0; for(int h=0;h<c->ngram_heads;h++) if(m->ple_head_offset[h]+m->ple_head_vocab[h]>need) need=m->ple_head_offset[h]+m->ple_head_vocab[h];- Q38_NEED(m->ple_part_start[m->ple_part_count]>=need,"PLE table rows %lld < required %lld",(long long)m->ple_part_start[m->ple_part_count],(long long)need);-}--static void q38_alloc_state(Model *m) {- Cfg *c=&m->c;- m->DN_rec=(float**)calloc((size_t)c->layers,sizeof(float*));- m->DN_conv=(float**)calloc((size_t)c->layers,sizeof(float*));- m->K=(float**)calloc((size_t)c->layers,sizeof(float*));- m->V=(float**)calloc((size_t)c->layers,sizeof(float*));- m->IK=(float**)calloc((size_t)c->layers,sizeof(float*));- if(!m->DN_rec||!m->DN_conv||!m->K||!m->V||!m->IK){fprintf(stderr,"OOM model state metadata\n");exit(1);}- for(int i=m->range_begin;i<m->range_end;i++) if(!c->is_attn[i]) {- m->DN_rec[i]=(float*)calloc((size_t)c->dn_vheads*c->dn_kdim*c->dn_vdim,sizeof(float));- m->DN_conv[i]=(float*)calloc((size_t)c->dn_conv_dim*(c->dn_convk-1),sizeof(float));- if(!m->DN_rec[i]||!m->DN_conv[i]){fprintf(stderr,"OOM DeltaNet state\n");exit(1);}- }- if(c->ple_layer>=m->range_begin&&c->ple_layer<m->range_end){- m->ple_history=(int64_t*)calloc(2,sizeof(int64_t));- m->PLE_conv_state=(float*)calloc((size_t)c->hc_width*(c->ple_convk-1)*c->ngram_size,sizeof(float));- if(!m->ple_history||!m->PLE_conv_state){fprintf(stderr,"OOM PLE state\n");exit(1);}- }-}--/* ---- vision ------------------------------------------------------------- */--/* La torre e' in F32 residente: 27 blocchi da 1152 sono ~0.6 GB, che accanto ai- * 9.2 GiB dei pesi densi non cambia la classe di macchina. Gli esperti restano- * su disco; la torre no, perche' si usa una volta per immagine e non per token. */-static const float *q38_vis_tensor(Model *m,const char *suffix,int64_t expect) {- char nm[512];- snprintf(nm,sizeof nm,"%s.visual.%s",m->prefix,suffix);- if(!st_has(&m->S,nm)){- snprintf(nm,sizeof nm,"model.visual.%s",suffix);- if(!st_has(&m->S,nm)){fprintf(stderr,"vision tensor missing: %s\n",suffix);exit(1);}- }- st_tensor *t=st_find(&m->S,nm);- if(expect>0&&t->numel!=expect){- fprintf(stderr,"vision tensor %s has %lld values, expected %lld -- refusing\n",- nm,(long long)t->numel,(long long)expect);exit(1);- }- float *buf=(float*)malloc((size_t)t->numel*sizeof(float));- if(!buf){fprintf(stderr,"OOM loading %s\n",nm);exit(1);}- st_read_f32(&m->S,nm,buf,t->numel);- m->resident_weight_bytes+=(uint64_t)t->numel*sizeof(float);- return buf;-}--static void q38_vis_linear(Model *m,Q38Linear *l,const char *stem,int out,int in) {- char nm[512];- snprintf(nm,sizeof nm,"%s.weight",stem); l->w=q38_vis_tensor(m,nm,(int64_t)out*in);- snprintf(nm,sizeof nm,"%s.bias",stem); l->b=q38_vis_tensor(m,nm,out);- l->out=out; l->in=in;-}--static void q38_vis_norm(Model *m,Q38Norm *n,const char *stem,int width) {- char nm[512];- snprintf(nm,sizeof nm,"%s.weight",stem); n->w=q38_vis_tensor(m,nm,width);- snprintf(nm,sizeof nm,"%s.bias",stem); n->b=q38_vis_tensor(m,nm,width);-}--static void q38_load_vision(Model *m) {- Cfg *c=&m->c;- if(!c->vis_depth) return;- char probe[512];- snprintf(probe,sizeof probe,"%s.visual.pos_embed.weight",m->prefix);- if(!st_has(&m->S,probe)&&!st_has(&m->S,"model.visual.pos_embed.weight")){- /* config multimodale ma pesi assenti: e' un export solo-testo di un- * checkpoint multimodale. Spegnere la torre e dirlo e' meglio che- * rifiutare un modello che per il testo funziona benissimo. */- fprintf(stderr,"[qwen38] vision_config present but no visual weights; text only\n");- c->vis_depth=0; return;- }- Q38Vision *v=&m->vis;- memset(v,0,sizeof *v);- v->depth=c->vis_depth; v->hidden=c->vis_hidden; v->heads=c->vis_heads;- v->head_dim=c->vis_hidden/c->vis_heads; v->inter=c->vis_inter;- v->patch=c->vis_patch; v->merge=c->vis_merge; v->temporal=c->vis_temporal;- v->in_ch=c->vis_in_ch; v->out_hidden=c->vis_out_hidden;- v->num_pos=c->vis_num_pos; v->side=(int)(sqrt((double)c->vis_num_pos)+0.5);- v->eps=1e-6f;- if(v->side*v->side!=v->num_pos){- fprintf(stderr,"[qwen38] num_position_embeddings=%d is not a square grid -- refusing\n",- v->num_pos);exit(1);- }- int features=v->in_ch*v->temporal*v->patch*v->patch;- q38_vis_linear(m,&v->patch_embed,"patch_embed.proj",v->hidden,features);- v->pos_embed=q38_vis_tensor(m,"pos_embed.weight",(int64_t)v->num_pos*v->hidden);- v->blocks=(Q38VBlock*)calloc((size_t)v->depth,sizeof(Q38VBlock));- if(!v->blocks){fprintf(stderr,"OOM vision blocks\n");exit(1);}- for(int i=0;i<v->depth;i++){- char stem[160];- snprintf(stem,sizeof stem,"blocks.%d.norm1",i); q38_vis_norm(m,&v->blocks[i].norm1,stem,v->hidden);- snprintf(stem,sizeof stem,"blocks.%d.norm2",i); q38_vis_norm(m,&v->blocks[i].norm2,stem,v->hidden);- snprintf(stem,sizeof stem,"blocks.%d.attn.qkv",i); q38_vis_linear(m,&v->blocks[i].qkv,stem,3*v->hidden,v->hidden);- snprintf(stem,sizeof stem,"blocks.%d.attn.proj",i); q38_vis_linear(m,&v->blocks[i].proj,stem,v->hidden,v->hidden);- snprintf(stem,sizeof stem,"blocks.%d.mlp.linear_fc1",i); q38_vis_linear(m,&v->blocks[i].fc1,stem,v->inter,v->hidden);- snprintf(stem,sizeof stem,"blocks.%d.mlp.linear_fc2",i); q38_vis_linear(m,&v->blocks[i].fc2,stem,v->hidden,v->inter);- }- int wide=v->hidden*v->merge*v->merge;- q38_vis_norm(m,&v->merger_norm,"merger.norm",v->hidden);- q38_vis_linear(m,&v->merger_fc1,"merger.linear_fc1",wide,wide);- q38_vis_linear(m,&v->merger_fc2,"merger.linear_fc2",v->out_hidden,wide);- m->vis_ready=1;- fprintf(stderr,"[qwen38] vision tower: %d blocks, hidden %d, %d heads, patch %d, merge %d\n",- v->depth,v->hidden,v->heads,v->patch,v->merge);-}--/* Esegue la torre su un'immagine gia' preprocessata e prepara la mappa- * posizione->riga. `ids`/`n` sono i token del prompt: le righe vengono- * assegnate ai token immagine nell'ordine in cui compaiono. */-static int q38_vision_attach(Model *m,const float *patches,int grid_h,int grid_w,- const int *ids,int n) {- Cfg *c=&m->c;- if(!m->vis_ready) return -1;- int tokens=(grid_h*grid_w)/(c->vis_merge*c->vis_merge);- int slots=0;- for(int i=0;i<n;i++) if(ids[i]==c->image_token) slots++;- if(slots!=tokens){- /* Il numero di segnaposto nel prompt DEVE essere quello che la griglia- * produce. Se non lo e', il template e il preprocessing hanno visto due- * immagini diverse, e proseguire vorrebbe dire mettere i vettori giusti- * nelle posizioni sbagliate. */- fprintf(stderr,"[qwen38] prompt has %d image placeholders but the grid gives %d tokens\n",- slots,tokens);- return -1;- }- free(m->vis_rows); free(m->vis_map);- m->vis_rows=(float*)calloc((size_t)tokens*c->hidden,sizeof(float));- m->vis_map=(int*)malloc((size_t)n*sizeof(int));- if(!m->vis_rows||!m->vis_map){free(m->vis_rows);free(m->vis_map);- m->vis_rows=NULL;m->vis_map=NULL;return -1;}- if(q38_vision_forward(&m->vis,patches,grid_h,grid_w,m->vis_rows)!=tokens){- free(m->vis_rows);free(m->vis_map);m->vis_rows=NULL;m->vis_map=NULL;return -1;}- int next=0;- for(int i=0;i<n;i++) m->vis_map[i]=(ids[i]==c->image_token)?next++:-1;- m->vis_map_len=n; m->vis_rows_n=tokens;- return tokens;-}--static void q38_vision_detach(Model *m) {- free(m->vis_rows); free(m->vis_map);- m->vis_rows=NULL; m->vis_map=NULL; m->vis_map_len=0; m->vis_rows_n=0;-}--static void model_init_range(Model *m,const char *snap,int cap,int bits,- int layer_begin,int layer_end,int load_boundaries,- int allocate_state) {- (void)bits; memset(m,0,sizeof(*m)); double t0=now_s();- m->native_fp8=q38_env_bool("Q38_NATIVE_FP8",1);- m->native_bf16=q38_env_bool("Q38_NATIVE_BF16",1);- m->expert_prefetch=q38_env_bool("Q38_EXPERT_PREFETCH",1);- m->expert_parallel_reads=q38_env_bool("Q38_EXPERT_PARALLEL_READS",1);- m->prefill_batch=q38_env_bool("Q38_PREFILL_BATCH",1);- q38_load_cfg(&m->c,snap); q38_validate_cfg(&m->c); st_init(&m->S,snap);- Cfg *c=&m->c; char nm[320];- if(st_has(&m->S,"model.language_model.embed_tokens.weight")) snprintf(m->prefix,sizeof m->prefix,"model.language_model");- else if(st_has(&m->S,"model.embed_tokens.weight")) snprintf(m->prefix,sizeof m->prefix,"model");- else {fprintf(stderr,"checkpoint has no Qwen4-Exp text embedding\n");exit(1);}- if(layer_end==0) layer_end=c->layers;- if(layer_begin<0||layer_begin>=layer_end||layer_end>c->layers){fprintf(stderr,"invalid Qwen3.8 layer range [%d,%d)\n",layer_begin,layer_end);exit(1);}- m->range_begin=layer_begin; m->range_end=layer_end;- if(load_boundaries){- snprintf(nm,sizeof nm,"%s.embed_tokens.weight",m->prefix); m->embed=q38_load_weight(m,nm,c->vocab,c->hidden);- m->lm_head=q38_load_weight(m,"lm_head.weight",c->vocab,c->hidden);- q38_load_gr(m,&m->final_gr,-1,NULL,0);- }- m->L=(Layer*)calloc((size_t)c->layers,sizeof(Layer));- m->cache=(LCache*)calloc((size_t)c->layers,sizeof(LCache));- m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers,- sizeof(*m->expert_scales));- if(!m->L||!m->cache||!m->expert_scales){fprintf(stderr,"OOM model metadata\n");exit(1);}- for(int i=layer_begin;i<layer_end;i++){- Layer *l=&m->L[i]; q38_load_gr(m,&l->attn_gr,i,"attn_hyper_connection",1); q38_load_gr(m,&l->mlp_gr,i,"mlp_hyper_connection",1);- #define WLD(field,suf,o,in) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_weight(m,nm,(o),(in))- #define VLD(field,suf,n) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_tensor(m,nm,(n))- WLD(router,"mlp.gate.weight",c->experts,c->hidden);- WLD(sh_g,"mlp.shared_expert.gate_proj.weight",c->shared_inter,c->hidden);- WLD(sh_u,"mlp.shared_expert.up_proj.weight",c->shared_inter,c->hidden);- WLD(sh_d,"mlp.shared_expert.down_proj.weight",c->hidden,c->shared_inter);- VLD(sh_gate,"mlp.shared_expert_gate.weight",c->hidden);- if(c->is_attn[i]){- WLD(q,"self_attn.q_proj.weight",c->q_heads*c->head_dim*2,c->hidden);- WLD(k,"self_attn.k_proj.weight",c->kv_heads*c->head_dim,c->hidden);- WLD(v,"self_attn.v_proj.weight",c->kv_heads*c->head_dim,c->hidden);- WLD(o,"self_attn.o_proj.weight",c->hidden,c->q_heads*c->head_dim);- VLD(qn,"self_attn.q_norm.weight",c->head_dim);VLD(kn,"self_attn.k_norm.weight",c->head_dim);- WLD(idx_qk,"self_attn.indexer.index_qk_proj.weight",(c->idx_qheads+c->idx_kheads)*c->idx_dim,c->hidden);- VLD(idx_qn,"self_attn.indexer.q_layernorm.weight",c->idx_dim);VLD(idx_kn,"self_attn.indexer.k_layernorm.weight",c->idx_dim);- } else {- int vd=c->dn_vheads*c->dn_vdim;- WLD(dn_qkv,"linear_attn.in_proj_qkv.weight",c->dn_conv_dim,c->hidden);- WLD(dn_z,"linear_attn.in_proj_z.weight",vd,c->hidden);- WLD(dn_b,"linear_attn.in_proj_b.weight",c->dn_vheads,c->hidden);- WLD(dn_a,"linear_attn.in_proj_a.weight",c->dn_vheads,c->hidden);- VLD(dn_conv,"linear_attn.conv1d.weight",(int64_t)c->dn_conv_dim*c->dn_convk);- VLD(dn_dtbias,"linear_attn.dt_bias",c->dn_vheads);VLD(dn_alog,"linear_attn.A_log",c->dn_vheads);- VLD(dn_norm,"linear_attn.norm.weight",c->dn_vdim);- WLD(dn_out,"linear_attn.out_proj.weight",c->hidden,vd);- }- #undef WLD- #undef VLD- LCache *lc=&m->cache[i]; lc->cap=cap; lc->slots=(Slot*)calloc((size_t)cap,sizeof(Slot)); lc->by_expert=(int*)malloc((size_t)c->experts*sizeof(int));- if(!lc->slots||!lc->by_expert){fprintf(stderr,"OOM expert cache\n");exit(1);} for(int e=0;e<c->experts;e++)lc->by_expert[e]=-1;- }- if(c->ple_layer>=layer_begin&&c->ple_layer<layer_end) q38_load_ple(m,&m->L[c->ple_layer]);- /* La torre solo quando il motore possiede la sequenza intera: uno shard che- * ospita solo alcuni layer non ha da fare niente con le immagini, e- * caricarla la' sarebbe mezzo giga per nulla. */- if(load_boundaries&&q38_env_bool("Q38_VISION",1)) q38_load_vision(m);- if(allocate_state) q38_alloc_state(m);- m->dense_load_s=now_s()-t0;- fprintf(stderr,"[qwen38] native text weights: prefix=%s, %d layers, PLE=%d, cache=%d/layer, "- "FP8=%s, BF16=%s (resident matrices %.2f GiB)\n",m->prefix,c->layers,c->ple_layer,cap,- m->native_fp8?"native":"expanded-f32",m->native_bf16?"native":"expanded-f32",- m->resident_weight_bytes/1073741824.0);-}--static void model_init(Model *m,const char *snap,int cap,int bits) {- model_init_range(m,snap,cap,bits,0,0,1,1);-}--static void q38_gr_read(Model *m,const GatedResidual *g,const float *hyper,- int S,float *mixed,float *inject) {- const Cfg *c=&m->c;- int H=c->hidden,W=c->hc_width,C=c->hc_count,R=c->hc_rank;- if(C<=0||H<=0||W!=C*H){fprintf(stderr,"invalid gated-residual width\n");exit(1);}- float *norm=falloc((int64_t)S*W),*low=falloc((int64_t)S*R),*mix=falloc((int64_t)S*W);- /* Makes the initialized-input invariant visible to aggressive interprocedural- * warning analysis; every element is overwritten by q38_rms0 below. */- memset(norm,0,(size_t)S*W*sizeof(float));- for(int s=0;s<S;s++) for(int b=0;b<C;b++)- q38_rms0(norm+(int64_t)s*W+(int64_t)b*H,hyper+(int64_t)s*W+(int64_t)b*H,g->norm+(int64_t)b*H,H,c->eps);- q38_dense_matmul(m,low,norm,&g->down,S,W,R);- for(int64_t z=0;z<(int64_t)S*R;z++) low[z]=q38_silu(low[z]/C);- q38_dense_matmul(m,mix,low,&g->up,S,R,W);- for(int s=0;s<S;s++) for(int d=0;d<H;d++){- float v=0.f;- for(int b=0;b<C;b++) v+=q38_sigmoid(mix[(int64_t)s*W+(int64_t)b*H+d])*norm[(int64_t)s*W+(int64_t)b*H+d];- mixed[(int64_t)s*H+d]=v/C;- }- if(inject){- q38_dense_matmul(m,inject,norm,&g->inject,S,W,C);- for(int64_t z=0;z<(int64_t)S*C;z++) inject[z]=2.f*q38_sigmoid(inject[z]/C);- }- free(norm);free(low);free(mix);-}--static void q38_gr_apply(const Cfg *c,float *hyper,const float *block,const float *inject,int S) {- int H=c->hidden,C=c->hc_count,W=c->hc_width;- for(int s=0;s<S;s++)for(int b=0;b<C;b++){- float a=inject[(int64_t)s*C+b];- for(int d=0;d<H;d++)hyper[(int64_t)s*W+(int64_t)b*H+d]+=a*block[(int64_t)s*H+d];- }-}--typedef struct {- st_tensor *tensor;- int expert, projection;-} Q38ScaleDesc;--static int q38_scale_desc_cmp(const void *left,const void *right) {- const Q38ScaleDesc *a=(const Q38ScaleDesc*)left;- const Q38ScaleDesc *b=(const Q38ScaleDesc*)right;- if(a->tensor->off<b->tensor->off)return -1;- if(a->tensor->off>b->tensor->off)return 1;- return 0;-}--static int q38_scale_group_span(Q38ScaleDesc *desc,int count,int *fd,- int64_t *begin,int64_t *nbytes) {- if(!desc||count<1||!fd||!begin||!nbytes)return -1;- qsort(desc,(size_t)count,sizeof(*desc),q38_scale_desc_cmp);- int group_fd=desc[0].tensor->fd;- int64_t first=desc[0].tensor->off,cursor=first;- for(int i=0;i<count;i++){- st_tensor *tensor=desc[i].tensor;- if(tensor->fd!=group_fd||tensor->off!=cursor||tensor->nbytes<=0||- tensor->nbytes>INT64_MAX-cursor)return -1;- cursor+=tensor->nbytes;- }- *fd=group_fd;*begin=first;*nbytes=cursor-first;return 0;-}--static void q38_decode_scale_tensor(float *out,const unsigned char *raw,- const st_tensor *tensor) {- if(tensor->dtype==2)memcpy(out,raw,(size_t)tensor->nbytes);- else for(int64_t i=0;i<tensor->numel;i++){- uint16_t half;memcpy(&half,raw+(size_t)i*sizeof(half),sizeof(half));- out[i]=tensor->dtype==0?bf16_to_f32(half):f16_to_f32(half);- }-}--/* The official checkpoint stores every layer's gate/up scale sidecars as one- * compact range and every down sidecar as another. Normalize both ranges to- * an expert-indexed F32 bank once, scattering by the numeric expert id rather- * than the checkpoint's lexical tensor order (0, 1, 10, ...). Variants that- * do not satisfy the compact-range invariant simply keep the established- * per-matrix loader. */-static int q38_prepare_expert_scale_bank(Model *m,int layer) {- Cfg *c=&m->c;- if(!m->native_fp8||layer<0||layer>=c->layers)return 0;- if(!m->expert_scales){- m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers,- sizeof(*m->expert_scales));- if(!m->expert_scales){fprintf(stderr,"OOM expert scale metadata\n");exit(1);}- }- Q38ExpertScaleCache *cache=&m->expert_scales[layer];- if(cache->ready)return cache->ready>0;- int64_t scale_count=fp8_nblk(c->inter)*fp8_nblk(c->hidden);- if(scale_count<1||(uint64_t)c->experts>SIZE_MAX/(3u*(uint64_t)scale_count*sizeof(float))){- fprintf(stderr,"invalid expert scale-bank geometry\n");exit(1);- }- Q38ScaleDesc *gate_up=(Q38ScaleDesc*)malloc((size_t)c->experts*2*sizeof(*gate_up));- Q38ScaleDesc *down=(Q38ScaleDesc*)malloc((size_t)c->experts*sizeof(*down));- if(!gate_up||!down){fprintf(stderr,"OOM expert scale descriptors\n");exit(1);}- int ngu=0,ndown=0;char suffix[192],name[320];- const char *projection[3]={"gate_proj","up_proj","down_proj"};- for(int expert=0;expert<c->experts;expert++)for(int kind=0;kind<3;kind++){- int rows=kind==2?c->hidden:c->inter;- int cols=kind==2?c->inter:c->hidden;- int length=snprintf(suffix,sizeof suffix,- "mlp.experts.%d.%s.weight_scale_inv",expert,projection[kind]);- if(length<0||(size_t)length>=sizeof suffix)goto incompatible;- q38_name(m,name,sizeof name,layer,suffix);- st_tensor *tensor=st_find(&m->S,name);- int64_t block_rows=fp8_nblk(rows),block_cols=fp8_nblk(cols);- if(!tensor||tensor->dtype<0||tensor->dtype>2||tensor->rank!=2||- tensor->shape[0]!=block_rows||tensor->shape[1]!=block_cols||- tensor->numel!=scale_count||- tensor->numel>INT64_MAX/st_dtype_esz(tensor->dtype)||- tensor->nbytes!=tensor->numel*st_dtype_esz(tensor->dtype))- goto incompatible;- Q38ScaleDesc item={tensor,expert,kind};- if(kind<2)gate_up[ngu++]=item;else down[ndown++]=item;- }- int gu_fd,down_fd;int64_t gu_begin,down_begin,gu_bytes,down_bytes;- if(q38_scale_group_span(gate_up,ngu,&gu_fd,&gu_begin,&gu_bytes)||- q38_scale_group_span(down,ndown,&down_fd,&down_begin,&down_bytes)||- (uint64_t)gu_bytes>SIZE_MAX||(uint64_t)down_bytes>SIZE_MAX)- goto incompatible;- float *values=(float*)malloc((size_t)c->experts*3*(size_t)scale_count*sizeof(float));- unsigned char *gu_raw=(unsigned char*)malloc((size_t)gu_bytes);- unsigned char *down_raw=(unsigned char*)malloc((size_t)down_bytes);- if(!values||!gu_raw||!down_raw){fprintf(stderr,"OOM resident expert scales\n");exit(1);}- double started=now_s();- st_read_range_raw_cap(&m->S,gu_fd,gu_begin,gu_bytes,gu_raw,gu_bytes,1,- "pread Qwen3.8 gate/up scales");- st_read_range_raw_cap(&m->S,down_fd,down_begin,down_bytes,down_raw,down_bytes,1,- "pread Qwen3.8 down scales");- q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_scale_reads+=2;- for(int i=0;i<ngu;i++){- Q38ScaleDesc *item=&gate_up[i];- float *dst=values+((int64_t)item->expert*3+item->projection)*scale_count;- q38_decode_scale_tensor(dst,gu_raw+(item->tensor->off-gu_begin),item->tensor);- }- for(int i=0;i<ndown;i++){- Q38ScaleDesc *item=&down[i];- float *dst=values+((int64_t)item->expert*3+item->projection)*scale_count;- q38_decode_scale_tensor(dst,down_raw+(item->tensor->off-down_begin),item->tensor);- }- free(gu_raw);free(down_raw);free(gate_up);free(down);- cache->values=values;cache->scale_count=scale_count;cache->ready=1;- m->expert_scale_bytes+=(uint64_t)c->experts*3*(uint64_t)scale_count*sizeof(float);- return 1;-incompatible:- free(gate_up);free(down);cache->ready=-1;return 0;-}--static void q38_bind_borrowed_fp8(Q38Weight *weight,void *data,float *scales,- int rows,int cols) {- q38_weight_free(weight);- weight->data=data;weight->scales=scales;weight->rows=rows;weight->cols=cols;- weight->elements=(int64_t)rows*cols;- weight->scale_count=fp8_nblk(rows)*fp8_nblk(cols);- weight->kind=Q38_WEIGHT_FP8;-}--static void q38_bind_fp8_slot(Slot *slot,float *scales,int scale_count,- int hidden,int intermediate) {- int64_t matrix_bytes=(int64_t)hidden*intermediate;- if(matrix_bytes<1||matrix_bytes>INT64_MAX/3||- (uint64_t)(matrix_bytes*3)>SIZE_MAX||- scale_count!=fp8_nblk(hidden)*fp8_nblk(intermediate)){- fprintf(stderr,"invalid native FP8 expert slab geometry\n");exit(1);- }- q38_weight_free(&slot->gate);q38_weight_free(&slot->up);- q38_weight_free(&slot->down);- int64_t slab_bytes=matrix_bytes*3;- if(!slot->fp8_slab||slot->fp8_slab_bytes!=slab_bytes){- void *replacement=realloc(slot->fp8_slab,(size_t)slab_bytes);- if(!replacement){fprintf(stderr,"OOM native FP8 expert slab\n");exit(1);}- slot->fp8_slab=replacement;slot->fp8_slab_bytes=slab_bytes;- }- unsigned char *raw=(unsigned char*)slot->fp8_slab;- q38_bind_borrowed_fp8(&slot->gate,raw,scales,intermediate,hidden);- q38_bind_borrowed_fp8(&slot->up,raw+matrix_bytes,scales+scale_count,- intermediate,hidden);- q38_bind_borrowed_fp8(&slot->down,raw+2*matrix_bytes,scales+2*scale_count,- hidden,intermediate);-}--static int q38_native_fp8_expert_tensors(Model *m,int layer,int expert,- st_tensor *weight[3]) {- if(!m->native_fp8)return 0;- Cfg *c=&m->c;const char *projection[3]={"gate_proj","up_proj","down_proj"};- int rows[3]={c->inter,c->inter,c->hidden};- int cols[3]={c->hidden,c->hidden,c->inter};- char suffix[192],name[320];- for(int kind=0;kind<3;kind++){- int length=snprintf(suffix,sizeof suffix,"mlp.experts.%d.%s.weight",- expert,projection[kind]);- if(length<0||(size_t)length>=sizeof suffix)return 0;- q38_name(m,name,sizeof name,layer,suffix);weight[kind]=st_find(&m->S,name);- if(!weight[kind]||weight[kind]->dtype!=4||weight[kind]->rank!=2||- weight[kind]->shape[0]!=rows[kind]||weight[kind]->shape[1]!=cols[kind]||- weight[kind]->nbytes!=(int64_t)rows[kind]*cols[kind])return 0;- }- if(weight[0]->fd!=weight[1]->fd||- weight[0]->off>INT64_MAX-weight[0]->nbytes||- weight[1]->off!=weight[0]->off+weight[0]->nbytes)return 0;- return 1;-}--static void q38_load_native_fp8_ranges(Model *m,int layer,int expert,Slot *slot,- st_tensor *weight[3]) {- Cfg *c=&m->c;- Q38ExpertScaleCache *cache=&m->expert_scales[layer];- float *scales=cache->values+(int64_t)expert*3*cache->scale_count;- /* Se i tre intervalli sono mappati (COLI_MAP_EXPERTS=1), lo slot li PUNTA- * invece di copiarli: niente slab, niente 14 MB per miss. */- {- const uint8_t *pg=(const uint8_t*)st_map_shard_range(weight[0]->fd,weight[0]->off,weight[0]->nbytes);- const uint8_t *pu=(const uint8_t*)st_map_shard_range(weight[1]->fd,weight[1]->off,weight[1]->nbytes);- const uint8_t *pd=(const uint8_t*)st_map_shard_range(weight[2]->fd,weight[2]->off,weight[2]->nbytes);- if(pg&&pu&&pd){- int sc=(int)cache->scale_count;- q38_bind_borrowed_fp8(&slot->gate,(void*)pg,scales,c->inter,c->hidden);- q38_bind_borrowed_fp8(&slot->up,(void*)pu,scales+sc,c->inter,c->hidden);- q38_bind_borrowed_fp8(&slot->down,(void*)pd,scales+2*sc,c->hidden,c->inter);- if(slot->fp8_slab){free(slot->fp8_slab);slot->fp8_slab=NULL;slot->fp8_slab_bytes=0;}- return;- }- }- q38_bind_fp8_slot(slot,scales,(int)cache->scale_count,c->hidden,c->inter);- int64_t pair_bytes=weight[0]->nbytes+weight[1]->nbytes;- unsigned char *raw=(unsigned char*)slot->fp8_slab;- st_read_range_raw_cap(&m->S,weight[0]->fd,weight[0]->off,pair_bytes,- raw,pair_bytes,1,"pread Qwen3.8 gate/up expert");- st_read_range_raw_cap(&m->S,weight[2]->fd,weight[2]->off,weight[2]->nbytes,- raw+pair_bytes,slot->fp8_slab_bytes-pair_bytes,1,- "pread Qwen3.8 down expert");-}--static int q38_try_load_native_fp8_expert(Model *m,int layer,int expert,Slot *slot) {- st_tensor *weight[3];- if(!q38_native_fp8_expert_tensors(m,layer,expert,weight)||- !q38_prepare_expert_scale_bank(m,layer))return 0;- double started=now_s();- q38_load_native_fp8_ranges(m,layer,expert,slot,weight);- q38_tm_add(m,Q38_TM_EXPERT_READ,started);- m->expert_weight_reads+=2;m->expert_pair_reads++;- return 1;-}--static void q38_prefetch_native_fp8_experts(Model *m,int layer,- const int *experts,int count) {- if(!m->expert_prefetch||!m->native_fp8||!experts||count<1||- !q38_prepare_expert_scale_bank(m,layer))return;- LCache *cache=&m->cache[layer];- for(int index=0;index<count;index++){- int expert=experts[index];st_tensor *weight[3];- if(expert<0||expert>=m->c.experts||cache->by_expert[expert]>=0||- !q38_native_fp8_expert_tensors(m,layer,expert,weight))continue;- posix_fadvise(weight[0]->fd,weight[0]->off,- weight[0]->nbytes+weight[1]->nbytes,POSIX_FADV_WILLNEED);- posix_fadvise(weight[2]->fd,weight[2]->off,weight[2]->nbytes,- POSIX_FADV_WILLNEED);- m->expert_prefetch_ranges+=2;- }-}--static void q38_load_fp8_expert_weight(Model *m,const char *wn,const char *sn,- Q38Weight *out,int O,int I) {- st_tensor *w=st_find(&m->S,wn),*sc=st_find(&m->S,sn);- int nb_o=(O+127)/128,nb_i=(I+127)/128;- if(!w||w->dtype!=4||w->rank!=2||w->shape[0]!=O||w->shape[1]!=I||- w->nbytes!=(int64_t)O*I||!sc||sc->dtype<0||sc->dtype>2||- sc->rank!=2||sc->shape[0]!=nb_o||sc->shape[1]!=nb_i||- sc->numel!=(int64_t)nb_o*nb_i){- fprintf(stderr,"invalid block-FP8 expert matrix %s / %s\n",wn,sn);exit(1);- }- if(m->native_fp8){- q38_weight_reserve(out,Q38_WEIGHT_FP8,O,I);- double started=now_s();- st_read_raw_cap(&m->S,wn,out->data,w->nbytes,1);- st_read_f32(&m->S,sn,out->scales,1);- q38_tm_add(m,Q38_TM_EXPERT_READ,started);- m->expert_weight_reads++;m->expert_scale_reads++;- return;- }- q38_weight_reserve(out,Q38_WEIGHT_F32,O,I);- uint8_t *raw=(uint8_t*)malloc((size_t)w->nbytes);float *scale=falloc(sc->numel);- if(!raw){fprintf(stderr,"OOM FP8 expert staging\n");exit(1);}- double started=now_s();- st_read_raw_cap(&m->S,wn,raw,w->nbytes,1);st_read_f32(&m->S,sn,scale,1);- q38_tm_add(m,Q38_TM_EXPERT_READ,started);- m->expert_weight_reads++;m->expert_scale_reads++;started=now_s();- float *decoded=(float*)out->data;- #pragma omp parallel for schedule(static)- for(int o=0;o<O;o++)for(int i=0;i<I;i++)- decoded[(int64_t)o*I+i]=e4m3_decode(raw[(int64_t)o*I+i])*- scale[(o/128)*nb_i+i/128];- q38_tm_add(m,Q38_TM_FP8_EXPAND,started);- free(raw);free(scale);-}--static void q38_load_expert_weight(Model *m,const char *name,Q38Weight *out,- int O,int I) {- st_tensor *tensor=st_find(&m->S,name);- if(!tensor||tensor->rank!=2||tensor->shape[0]!=O||tensor->shape[1]!=I||- tensor->numel!=(int64_t)O*I||tensor->dtype<0||tensor->dtype>2){- fprintf(stderr,"invalid expert matrix %s\n",name);exit(1);- }- Q38WeightKind kind=m->native_bf16&&tensor->dtype==0?Q38_WEIGHT_BF16:Q38_WEIGHT_F32;- q38_weight_reserve(out,kind,O,I);double started=now_s();- if(kind==Q38_WEIGHT_BF16)- st_read_raw_cap(&m->S,name,out->data,tensor->nbytes,1);- else st_read_f32(&m->S,name,(float*)out->data,1);- q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;-}--static void q38_load_expert_slice(Model *m,const char *name,const st_tensor *tensor,- int64_t element_offset,Q38Weight *out,int O,int I) {- if(!tensor||tensor->dtype<0||tensor->dtype>2||element_offset<0||- element_offset>tensor->numel-(int64_t)O*I){- fprintf(stderr,"invalid expert slice %s\n",name);exit(1);- }- Q38WeightKind kind=m->native_bf16&&tensor->dtype==0?Q38_WEIGHT_BF16:Q38_WEIGHT_F32;- q38_weight_reserve(out,kind,O,I);double started=now_s();- if(kind==Q38_WEIGHT_BF16){- int64_t bytes=(int64_t)O*I*(int64_t)sizeof(uint16_t);- st_read_slice_raw_cap(&m->S,name,element_offset*(int64_t)sizeof(uint16_t),- bytes,out->data,bytes,1);- } else st_read_slice_f32(&m->S,name,element_offset,(int64_t)O*I,(float*)out->data,1);- q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;-}--static void q38_load_expert(Model *m,int layer,int eid,Slot *s) {- Cfg *c=&m->c; int H=c->hidden,I=c->inter; char nm[320],sn[340];- q38_name(m,nm,sizeof nm,layer,"mlp.experts.gate_up_proj");- if(st_has(&m->S,nm)){- st_tensor *t=st_find(&m->S,nm);- if(t->rank!=3||t->shape[0]!=c->experts||t->shape[1]!=2*I||t->shape[2]!=H){fprintf(stderr,"invalid %s\n",nm);exit(1);}- int64_t base=(int64_t)eid*2*I*H;- q38_load_expert_slice(m,nm,t,base,&s->gate,I,H);- q38_load_expert_slice(m,nm,t,base+(int64_t)I*H,&s->up,I,H);- q38_name(m,nm,sizeof nm,layer,"mlp.experts.down_proj"); t=st_find(&m->S,nm);- if(!t||t->rank!=3||t->shape[0]!=c->experts||t->shape[1]!=H||t->shape[2]!=I){fprintf(stderr,"invalid %s\n",nm);exit(1);}- q38_load_expert_slice(m,nm,t,(int64_t)eid*H*I,&s->down,H,I);- return;- }- if(q38_try_load_native_fp8_expert(m,layer,eid,s))return;- const char *kind[3]={"gate_proj","up_proj","down_proj"};Q38Weight *dst[3]={&s->gate,&s->up,&s->down};- int os[3]={I,I,H},is[3]={H,H,I};- for(int k=0;k<3;k++){- char suf[192]; snprintf(suf,sizeof suf,"mlp.experts.%d.%s.weight",eid,kind[k]);q38_name(m,nm,sizeof nm,layer,suf);- st_tensor *t=st_find(&m->S,nm); if(!t){fprintf(stderr,"missing %s\n",nm);exit(1);}- if(t->dtype==4){snprintf(sn,sizeof sn,"%s_scale_inv",nm);q38_load_fp8_expert_weight(m,nm,sn,dst[k],os[k],is[k]);}- else q38_load_expert_weight(m,nm,dst[k],os[k],is[k]);- }-}--/* One byte per expert: routed in this turn or not. The dashboard's Brain tab- * reads it as the HITS bitmap after every turn (serve_hits in qwen38.c),- * cleared there. Marked on both lookup paths, single and batched. */-static pthread_mutex_t g_q38_ehit_mx=PTHREAD_MUTEX_INITIALIZER;-static void q38_ehit_mark(Model *m,int layer,int eid) {- const Cfg *c=&m->c;- /* The first touch can come from a parallel region (qwen36: the tier- * warmstart's omp loop calls expert_get from twelve threads at once):- * one thread published m->ehit while it was still filling the rows and- * a sibling dereferenced m->ehit[layer] == NULL -- SIGSEGV in about one- * run in twelve on a CUDA warmstart. Build the table privately, publish- * it once under a lock (double-checked), and read it with acquire order. */- uint8_t **ehit=__atomic_load_n(&m->ehit,__ATOMIC_ACQUIRE);- if(!ehit){- pthread_mutex_lock(&g_q38_ehit_mx);- ehit=m->ehit;- if(!ehit){- ehit=(uint8_t**)calloc((size_t)c->layers,sizeof(uint8_t*));- for(int i=0;i<c->layers;i++)ehit[i]=(uint8_t*)calloc((size_t)c->experts,1);- __atomic_store_n(&m->ehit,ehit,__ATOMIC_RELEASE);- }- pthread_mutex_unlock(&g_q38_ehit_mx);- }- if(layer>=0&&layer<c->layers&&eid>=0&&eid<c->experts)ehit[layer][eid]=1;-}-static Slot *q38_expert_get(Model *m,int layer,int eid) {- q38_ehit_mark(m,layer,eid);- LCache *lc=&m->cache[layer]; int si=lc->by_expert[eid];- if(si>=0){m->hits++;lc->slots[si].used=++m->clock;return &lc->slots[si];}- m->miss++; Slot *s;- if(lc->n<lc->cap){s=&lc->slots[lc->n++];s->eid=-1;}- else {- int victim=0;for(int i=1;i<lc->n;i++)if(lc->slots[i].used<lc->slots[victim].used)victim=i;- s=&lc->slots[victim];if(s->eid>=0)lc->by_expert[s->eid]=-1;- }- s->eid=-1;q38_load_expert(m,layer,eid,s);s->eid=eid;s->used=++m->clock;lc->by_expert[eid]=(int)(s-lc->slots);return s;-}-+ uint64_t resident_weight_bytes;+ double dense_load_s;+ /* vision. `vis_map` mappa la posizione ASSOLUTA nella sequenza alla riga di+ * `vis_rows`, oppure -1. Assoluta e non relativa al chunk: il prefill arriva+ * a pezzi, e un indice relativo darebbe l'immagine sbagliata al secondo+ * pezzo senza che niente protesti. */+ /* PLE prefetto: le righe gia' lette per il chunk in corso, o NULL. */+ float *ple_pref; int ple_pref_rows;+ Q38Vision vis;+ int vis_ready;+ float *vis_rows;+ int *vis_map, vis_map_len, vis_rows_n;+ Q38Timers timers;+} Model;++static float *g_last_logit;+static int g_capture_last_logit;++static double now_s(void) {+ struct timespec t;+ clock_gettime(CLOCK_MONOTONIC, &t);+ return (double)t.tv_sec + (double)t.tv_nsec * 1e-9;+}++static inline void q38_tm_add(Model *m,Q38Timer timer,double started) {+ m->timers.seconds[timer] += now_s() - started;+}++static Q38Timers q38_tm_delta(const Q38Timers *after,const Q38Timers *before) {+ Q38Timers delta={0};+ for(int i=0;i<Q38_TM_COUNT;i++)+ delta.seconds[i]=after->seconds[i]-before->seconds[i];+ delta.forwards=after->forwards-before->forwards;+ return delta;+}++#if defined(__APPLE__)+static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF,&r); return r.ru_maxrss/1073741824.0; }+#else+static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF,&r); return r.ru_maxrss/1048576.0; }+#endif++static float *falloc(int64_t n) {+ if (n < 0 || (uint64_t)n > SIZE_MAX / sizeof(float)) {+ fprintf(stderr, "invalid float allocation: %lld\n", (long long)n); exit(1);+ }+ float *p = (float *)malloc((size_t)n * sizeof(float));+ if (!p && n) { fprintf(stderr, "OOM allocating %lld floats\n", (long long)n); exit(1); }+ return p;+}++/* W is row-major [O,I], y=x@W^T. */+static void q38_matmul(float *y, const float *x, const float *W, int S, int I, int O) {+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) {+ const float *w = W + (int64_t)o * I;+ for (int s = 0; s < S; s++) {+ const float *xs = x + (int64_t)s * I;+ float a = 0.f;+ for (int i = 0; i < I; i++) a += xs[i] * w[i];+ y[(int64_t)s * O + o] = a;+ }+ }+}++static void q38_weight_free(Q38Weight *weight) {+ if(!weight)return;+ if(weight->owns_data)free(weight->data);+ if(weight->owns_scales)free(weight->scales);+ free(weight->q8); free(weight->q8sc);+ memset(weight,0,sizeof(*weight));+}++static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,int cols) {+ if(rows<=0||cols<=0||(int64_t)rows>INT64_MAX/cols){+ fprintf(stderr,"invalid weight geometry [%d,%d]\n",rows,cols);exit(1);+ }+ int64_t elements=(int64_t)rows*cols;+ int64_t scales=kind==Q38_WEIGHT_FP8?fp8_nblk(rows)*fp8_nblk(cols):0;+ if(weight->kind==kind&&weight->rows==rows&&weight->cols==cols&&+ weight->owns_data&&weight->data&&+ (!scales||(weight->owns_scales&&weight->scales)))return;+ q38_weight_free(weight);+ size_t element_size=kind==Q38_WEIGHT_F32?sizeof(float):+ kind==Q38_WEIGHT_BF16?sizeof(uint16_t):+ kind==Q38_WEIGHT_FP8?sizeof(uint8_t):0;+ if(!element_size||(uint64_t)elements>SIZE_MAX/element_size||+ (scales&&(uint64_t)scales>SIZE_MAX/sizeof(float))){+ fprintf(stderr,"unsupported or oversized weight geometry [%d,%d] kind=%d\n",+ rows,cols,(int)kind);exit(1);+ }+ weight->data=malloc((size_t)elements*element_size);+ if(scales)weight->scales=(float*)malloc((size_t)scales*sizeof(float));+ if(!weight->data||(scales&&!weight->scales)){+ fprintf(stderr,"OOM allocating weight [%d,%d] kind=%d\n",rows,cols,(int)kind);exit(1);+ }+ weight->rows=rows;weight->cols=cols;weight->elements=elements;+ weight->scale_count=scales;weight->kind=kind;+ weight->owns_data=1;weight->owns_scales=scales?1u:0u;+}++static uint64_t q38_weight_bytes(const Q38Weight *weight) {+ uint64_t element_size=weight->kind==Q38_WEIGHT_F32?sizeof(float):+ weight->kind==Q38_WEIGHT_BF16?sizeof(uint16_t):+ weight->kind==Q38_WEIGHT_FP8?sizeof(uint8_t):0;+ return (uint64_t)weight->elements*element_size++ (uint64_t)weight->scale_count*sizeof(float);+}++/* Native BF16 storage with FP32 activations and accumulation. This deliberately+ * does not round activations to BF16 or use BF16 dot-product instructions: it is+ * the storage-equivalent form of the existing st_read_f32 reference. */+static void q38_matmul_bf16(float *y,const float *x,const uint16_t *W,+ int S,int I,int O) {+ #pragma omp parallel for schedule(static)+ for(int o=0;o<O;o++){+ const uint16_t *w=W+(int64_t)o*I;+ for(int s=0;s<S;s++){+ const float *xs=x+(int64_t)s*I;float a=0.f;+ for(int i=0;i<I;i++)a+=xs[i]*bf16_to_f32(w[i]);+ y[(int64_t)s*O+o]=a;+ }+ }+}++static void q38_weight_matmul(float *y,const float *x,const Q38Weight *weight,+ int S,int I,int O) {+ /* A matrix the tier placed in VRAM (q38_trunk_place) answers a decode+ * GEMV from there; prefill rows and any failure take the CPU path below,+ * so the BF16 copy stays the reference for everything but S == 1. */+ if(S==1&&weight&&weight->gpu&&weight->rows==O&&weight->cols==I&&+ qt_dense_matmul(weight->gpu-1,y,x,I,O))return;+ if(S==1&&weight&&weight->q8&&weight->rows==O&&weight->cols==I){+ /* the int8 rows the GPU would hold, computed here: what the trunk+ * quantization alone does to the output, GPU or not */+ const int8_t *q=weight->q8; const float *sc=weight->q8sc;+ #pragma omp parallel for schedule(static)+ for(int o=0;o<O;o++){+ const int8_t *w=q+(size_t)o*I; float a=0.f;+ for(int i=0;i<I;i++)a+=x[i]*(float)w[i];+ y[o]=a*sc[o];+ }+ return;+ }+ if(!weight||weight->rows!=O||weight->cols!=I||!weight->data){+ fprintf(stderr,"invalid matmul weight: have [%d,%d] kind=%d, need [%d,%d]\n",+ weight?weight->rows:0,weight?weight->cols:0,+ weight?(int)weight->kind:0,O,I);exit(1);+ }+ if(weight->kind==Q38_WEIGHT_F32)+ q38_matmul(y,x,(const float*)weight->data,S,I,O);+ else if(weight->kind==Q38_WEIGHT_BF16)+ q38_matmul_bf16(y,x,(const uint16_t*)weight->data,S,I,O);+ else if(weight->kind==Q38_WEIGHT_FP8&&weight->scales)+ matmul_fp8(y,x,(const uint8_t*)weight->data,weight->scales,S,I,O);+ else {fprintf(stderr,"unsupported matmul weight kind %d\n",(int)weight->kind);exit(1);}+}++static void q38_weight_row(const Q38Weight *weight,int row,float *out) {+ if(!weight||row<0||row>=weight->rows||!weight->data){+ fprintf(stderr,"invalid weight row %d\n",row);exit(1);+ }+ if(weight->kind==Q38_WEIGHT_F32)+ memcpy(out,(const float*)weight->data+(int64_t)row*weight->cols,+ (size_t)weight->cols*sizeof(float));+ else if(weight->kind==Q38_WEIGHT_BF16){+ const uint16_t *src=(const uint16_t*)weight->data+(int64_t)row*weight->cols;+ for(int i=0;i<weight->cols;i++)out[i]=bf16_to_f32(src[i]);+ } else {fprintf(stderr,"weight kind %d has no dense row view\n",(int)weight->kind);exit(1);}+}++static void q38_dense_matmul(Model *m,float *y,const float *x,const Q38Weight *weight,+ int S,int I,int O) {+ double started=now_s();+ q38_weight_matmul(y,x,weight,S,I,O);+ q38_tm_add(m,Q38_TM_DENSE_MATMUL,started);+}++static inline float q38_sigmoid(float x) {+ if (x >= 0.f) { float z=expf(-x); return 1.f/(1.f+z); }+ float z=expf(x); return z/(1.f+z);+}+static inline float q38_silu(float x) { return x * q38_sigmoid(x); }+static inline float q38_softplus(float x) { return x > 20.f ? x : log1pf(expf(x)); }++/* Qwen4-Exp RMSNorms are zero-centered: the learned scale is 1+weight. */+static void q38_rms0(float *out, const float *x, const float *w, int n, float eps) {+ double ss=0.0; for (int i=0;i<n;i++) ss+=(double)x[i]*x[i];+ float r=1.f/sqrtf((float)(ss/n)+eps);+ for (int i=0;i<n;i++) out[i]=x[i]*r*(1.f+w[i]);+}++/* DeltaNet's RMSNormGated is inherited from Qwen3-Next and is not zero-centered. */+static void q38_rmsg(float *out,const float *x,const float *gate,const float *w,+ int n,float eps,int sigmoid_gate) {+ double ss=0.0; for(int i=0;i<n;i++) ss+=(double)x[i]*x[i];+ float r=1.f/sqrtf((float)(ss/n)+eps);+ for(int i=0;i<n;i++) out[i]=x[i]*r*w[i]*(sigmoid_gate?q38_sigmoid(gate[i]):q38_silu(gate[i]));+}++static void q38_rope(float *x, int dim, int rotary_dim, int pos, float theta) {+ int half=rotary_dim/2;+ for(int i=0;i<half;i++) {+ float ang=(float)pos/powf(theta,(float)(2*i)/rotary_dim);+ float co=cosf(ang), si=sinf(ang), a=x[i], b=x[i+half];+ x[i]=a*co-b*si; x[i+half]=b*co+a*si;+ }+ (void)dim;+}++static jval *q38_obj(jval *o,const char *key) {+ jval *v=json_get(o,key);+ if(v&&v->t!=J_OBJ){fprintf(stderr,"config.json: %s must be an object\n",key);exit(1);}+ return v;+}+static double q38_num(jval *o,const char *key,double def,int required) {+ jval *v=json_get(o,key);+ if(v&&v->t==J_NUM) return v->num;+ if(v){fprintf(stderr,"config.json: %s must be numeric\n",key);exit(1);}+ if(required){fprintf(stderr,"config.json: missing numeric %s\n",key);exit(1);} return def;+}+/* JSON numbers are doubles in the small parser. Never narrow an unchecked+ * value to an int: besides accepting fractional dimensions, a huge finite+ * value can become implementation-defined before q38_validate_cfg sees it. */+static int q38_num_int(jval *o,const char *key,double def,int required,+ int min_value,int max_value) {+ double value=q38_num(o,key,def,required);+ if(!isfinite(value)||floor(value)!=value||value<(double)min_value||+ value>(double)max_value){+ fprintf(stderr,"config.json: %s must be an integer in [%d,%d]\n",+ key,min_value,max_value);exit(1);+ }+ return (int)value;+}+static int q38_derived_product(const char *name,int left,int right) {+ if(left<0||right<0||(uint64_t)left*(uint64_t)right>(uint64_t)INT_MAX){+ fprintf(stderr,"[qwen38 config] derived %s overflows int\n",name);exit(1);+ }+ return left*right;+}+static int q38_derived_sum(const char *name,int left,int right) {+ if(left<0||right<0||left>INT_MAX-right){+ fprintf(stderr,"[qwen38 config] derived %s overflows int\n",name);exit(1);+ }+ return left+right;+}+static int q38_bool(jval *o,const char *key,int def) {+ jval *v=json_get(o,key);+ if(v&&v->t!=J_BOOL){fprintf(stderr,"config.json: %s must be boolean\n",key);exit(1);}+ return v?v->boolean:def;+}+static const char *q38_string(jval *o,const char *key,const char *def) {+ jval *v=json_get(o,key);+ if(v&&v->t!=J_STR){fprintf(stderr,"config.json: %s must be a string\n",key);exit(1);}+ return v?v->str:def;+}+static void q38_require_string(jval *o,const char *key,const char *expected) {+ const char *value=q38_string(o,key,expected);+ if(strcmp(value,expected)){+ fprintf(stderr,"config.json: unsupported %s=%s (expected %s)\n",+ key,value,expected);exit(1);+ }+}+static void q38_require_present_string(jval *o,const char *key,+ const char *expected) {+ jval *value=json_get(o,key);+ if(!value||value->t!=J_STR||strcmp(value->str,expected)){+ fprintf(stderr,"config.json: %s must be explicitly set to %s\n",+ key,expected);exit(1);+ }+}+static void q38_require_bool(jval *o,const char *key,int expected) {+ int value=q38_bool(o,key,expected);+ if(value!=expected){+ fprintf(stderr,"config.json: unsupported %s=%s (expected %s)\n",key,+ value?"true":"false",expected?"true":"false");exit(1);+ }+}++static void q38_load_cfg(Cfg *c,const char *snap) {+ memset(c,0,sizeof(*c));+ char path[2048]; snprintf(path,sizeof path,"%s/config.json",snap);+ FILE *f=fopen(path,"rb"); if(!f){perror(path);exit(1);} fseek(f,0,SEEK_END);+ long n=ftell(f); fseek(f,0,SEEK_SET);+ if(n<0||n>(256L<<20)){fprintf(stderr,"invalid config size\n");exit(1);}+ char *buf=(char*)malloc((size_t)n+1),*arena=NULL;+ if(!buf||fread(buf,1,(size_t)n,f)!=(size_t)n){fprintf(stderr,"cannot read %s\n",path);exit(1);}+ fclose(f); buf[n]=0; jval *root=json_parse(buf,&arena);+ if(!root||root->t!=J_OBJ){fprintf(stderr,"invalid %s\n",path);exit(1);}+ jval *tc=q38_obj(root,"text_config"); if(!tc) tc=root;+ const char *mt=jstr(tc,"model_type");+ if(!mt||strcmp(mt,"qwen4_exp_text")){fprintf(stderr,"unsupported text model_type: %s\n",mt?mt:"(missing)");exit(1);}+ q38_require_string(tc,"hidden_act","silu");+ /* Upstream's omitted/None default resolves to hidden_act (SiLU), whereas+ * this implementation deliberately uses the released sigmoid gate. */+ q38_require_present_string(tc,"output_gate_type","sigmoid");+ q38_require_bool(tc,"attention_bias",0);+ q38_require_bool(tc,"tie_word_embeddings",0);+ c->hidden=q38_num_int(tc,"hidden_size",0,1,0,INT_MAX);+ c->layers=q38_num_int(tc,"num_hidden_layers",0,1,0,Q38_MAX_LAYERS);+ c->vocab=q38_num_int(tc,"vocab_size",0,1,0,INT_MAX);+ c->max_positions=q38_num_int(tc,"max_position_embeddings",0,1,0,INT_MAX);+ c->eos_id=q38_num_int(tc,"eos_token_id",-1,1,INT_MIN,INT_MAX);+ double eps=q38_num(tc,"rms_norm_eps",1e-6,0);+ if(!isfinite(eps)||eps<=0.0||eps>FLT_MAX){fprintf(stderr,"invalid rms_norm_eps\n");exit(1);}+ c->eps=(float)eps;+ jval *rp=q38_obj(tc,"rope_parameters");+ if(rp)q38_require_string(rp,"rope_type","default");+ double theta=rp?q38_num(rp,"rope_theta",10000,0):q38_num(tc,"rope_theta",10000,0);+ if(!isfinite(theta)||theta<=0.0||theta>FLT_MAX){fprintf(stderr,"invalid rope_theta\n");exit(1);}+ c->theta=(float)theta;+ c->hc_count=q38_num_int(tc,"hc_count",4,0,0,INT_MAX);+ c->hc_rank=q38_num_int(tc,"hc_lowrank",320,0,0,INT_MAX);+ c->hc_width=q38_derived_product("hc_width",c->hc_count,c->hidden);+ c->q_heads=q38_num_int(tc,"num_attention_heads",0,1,0,INT_MAX);+ c->kv_heads=q38_num_int(tc,"num_key_value_heads",0,1,0,INT_MAX);+ c->head_dim=q38_num_int(tc,"head_dim",0,1,0,INT_MAX);+ double partial=rp?q38_num(rp,"partial_rotary_factor",q38_num(tc,"partial_rotary_factor",1,0),0):q38_num(tc,"partial_rotary_factor",1,0);+ double rotary_product=(double)c->head_dim*partial;+ if(!isfinite(partial)||!isfinite(rotary_product)||rotary_product<0||+ rotary_product>(double)INT_MAX){+ fprintf(stderr,"[qwen38 config] derived rotary_dim overflows int\n");exit(1);+ }+ /* Upstream Qwen4ExpTextConfig derives this with Python int(), i.e. truncation+ * toward zero rather than rounding to the nearest dimension. */+ c->rotary_dim=(int)rotary_product;+ c->idx_qheads=q38_num_int(tc,"indexer_n_heads",0,1,0,INT_MAX);+ c->idx_kheads=q38_num_int(tc,"indexer_kv_heads",0,1,0,INT_MAX);+ c->idx_dim=q38_num_int(tc,"indexer_head_dim",0,1,0,INT_MAX);+ c->idx_budget=q38_num_int(tc,"indexer_budget",0,1,0,INT_MAX);+ c->idx_ratio=q38_num_int(tc,"indexer_compress_ratio",0,1,0,INT_MAX);+ c->experts=q38_num_int(tc,"num_experts",0,1,0,INT_MAX);+ c->topk=q38_num_int(tc,"num_experts_per_tok",0,1,0,INT_MAX);+ c->inter=q38_num_int(tc,"moe_intermediate_size",0,1,0,INT_MAX);+ c->shared_inter=q38_num_int(tc,"shared_expert_intermediate_size",0,1,0,INT_MAX);+ c->norm_topk=q38_bool(tc,"norm_topk_prob",1);+ c->dn_kheads=q38_num_int(tc,"linear_num_key_heads",0,1,0,INT_MAX);+ c->dn_vheads=q38_num_int(tc,"linear_num_value_heads",0,1,0,INT_MAX);+ c->dn_kdim=q38_num_int(tc,"linear_key_head_dim",0,1,0,INT_MAX);+ c->dn_vdim=q38_num_int(tc,"linear_value_head_dim",0,1,0,INT_MAX);+ c->dn_convk=q38_num_int(tc,"linear_conv_kernel_dim",0,1,0,INT_MAX);+ int dn_qk=q38_derived_product("deltanet_qk",c->dn_kheads,c->dn_kdim);+ dn_qk=q38_derived_product("deltanet_qk_twice",2,dn_qk);+ c->dn_conv_dim=q38_derived_sum("dn_conv_dim",dn_qk,+ q38_derived_product("deltanet_v",c->dn_vheads,c->dn_vdim));+ c->ple_dim=q38_num_int(tc,"ple_embed_dim",c->hidden,0,0,INT_MAX);+ c->ple_convk=q38_num_int(tc,"ple_conv_kernel_size",4,0,0,INT_MAX);+ c->ngram_size=q38_num_int(tc,"ngram_size",3,0,0,INT_MAX);+ c->heads_per_ngram=q38_num_int(tc,"heads_per_ngram",8,0,0,INT_MAX);+ c->ngram_heads=(c->ngram_size>0)?q38_derived_product("ngram_heads",c->ngram_size-1,c->heads_per_ngram):0;+ if(c->ngram_heads<=0){fprintf(stderr,"[qwen38 config] derived ngram_heads invalid\n");exit(1);}+ c->ngram_head_dim=c->ple_dim/c->ngram_heads;+ c->ngram_parts=q38_num_int(tc,"split_ngram_parts",1,0,0,INT_MAX);+ c->ple_layer=-1;+ jval *pl=json_get(tc,"ple_layer_ids");+ if(pl&&pl->t==J_ARR&&pl->len){+ if(pl->len!=1||pl->kids[0]->t!=J_NUM){fprintf(stderr,"only one PLE layer is supported\n");exit(1);}+ double raw=pl->kids[0]->num;+ if(!isfinite(raw)||floor(raw)!=raw||raw<1||raw>(double)INT_MAX){+ fprintf(stderr,"config.json: ple_layer_ids[0] must be a positive integer\n");exit(1);+ }+ int one_based=(int)raw;+ c->ple_layer=one_based-1;+ }+ c->is_attn=(uint8_t*)calloc((size_t)c->layers,1);+ jval *lt=json_get(tc,"layer_types");+ if(!lt||lt->t!=J_ARR||lt->len!=c->layers){fprintf(stderr,"config layer_types must have %d entries\n",c->layers);exit(1);}+ for(int i=0;i<c->layers;i++){+ const char *s=lt->kids[i]->t==J_STR?lt->kids[i]->str:NULL;+ if(!s){fprintf(stderr,"invalid layer_types[%d]\n",i);exit(1);}+ if(!strcmp(s,"linear_attention")) c->is_attn[i]=0;+ else if(!strcmp(s,"full_attention")||!strcmp(s,"qwen_sparse_attention")) c->is_attn[i]=1;+ else {fprintf(stderr,"unsupported layer type %s\n",s);exit(1);}+ }+ /* Vision: opzionale. Un checkpoint di solo testo non ha vision_config, e in+ * quel caso la torre resta spenta invece di rifiutare il modello. La+ * geometria viene dal file, non da costanti qui: una torre di misura diversa+ * deve fallire dicendo cosa non torna, non leggere pesi della misura+ * sbagliata. */+ {+ jval *vc = q38_obj(root, "vision_config");+ if (vc) {+ c->vis_depth = q38_num_int(vc,"depth",0,1,1,1024);+ c->vis_hidden = q38_num_int(vc,"hidden_size",0,1,1,65536);+ c->vis_heads = q38_num_int(vc,"num_heads",0,1,1,1024);+ c->vis_inter = q38_num_int(vc,"intermediate_size",0,1,1,262144);+ c->vis_patch = q38_num_int(vc,"patch_size",16,0,1,512);+ c->vis_merge = q38_num_int(vc,"spatial_merge_size",2,0,1,8);+ c->vis_temporal = q38_num_int(vc,"temporal_patch_size",2,0,1,8);+ c->vis_in_ch = q38_num_int(vc,"in_channels",3,0,1,8);+ c->vis_out_hidden = q38_num_int(vc,"out_hidden_size",c->hidden,0,1,65536);+ c->vis_num_pos = q38_num_int(vc,"num_position_embeddings",0,1,1,1<<20);+ c->image_token = q38_num_int(root,"image_token_id",-1,0,0,INT_MAX);+ }+ }+ json_free(root); free(buf); free(arena);+}++#define Q38_NEED(x,...) do{if(!(x)){fprintf(stderr,"[qwen38 config] ");fprintf(stderr,__VA_ARGS__);fprintf(stderr," -- refusing\n");exit(1);}}while(0)+static void q38_validate_cfg(const Cfg *c) {+ Q38_NEED(c->hidden>0&&c->hidden<=65536,"hidden_size=%d",c->hidden);+ if (c->vis_depth) {+ /* La torre proietta direttamente nello spazio del testo: se le due+ * dimensioni non coincidono i token immagine finirebbero nel posto+ * giusto con i numeri sbagliati, che e' peggio di un rifiuto. */+ Q38_NEED(c->vis_out_hidden==c->hidden,+ "vision out_hidden_size=%d but text hidden_size=%d",+ c->vis_out_hidden,c->hidden);+ Q38_NEED(c->vis_hidden%c->vis_heads==0,+ "vision hidden_size=%d not divisible by num_heads=%d",+ c->vis_hidden,c->vis_heads);+ Q38_NEED(c->image_token>=0&&c->image_token<c->vocab,+ "image_token_id=%d outside vocabulary",c->image_token);+ }+ Q38_NEED(c->layers>0&&c->layers<=Q38_MAX_LAYERS,"layers=%d",c->layers);+ Q38_NEED(c->vocab>0&&c->max_positions>0&&+ c->max_positions<=QWEN38_ATTN_MAX_CTX,"vocab/context invalid");+ Q38_NEED(isfinite(c->eps)&&c->eps>0.f&&isfinite(c->theta)&&c->theta>0.f,"RoPE/norm constants invalid");+ Q38_NEED(c->eos_id>=0&&c->eos_id<c->vocab,"eos=%d",c->eos_id);+ Q38_NEED(c->hc_count>1&&c->hc_count<=16&&c->hc_rank>0&&+ c->hc_rank<=65536&&c->hc_width>0,"gated residual dimensions invalid");+ Q38_NEED(c->q_heads>0&&c->kv_heads>0&&c->q_heads%c->kv_heads==0,"attention heads invalid");+ Q38_NEED(c->head_dim>0&&c->rotary_dim>0&&!(c->rotary_dim&1)&&c->rotary_dim<=c->head_dim,"RoPE dimensions invalid");+ Q38_NEED(c->head_dim<=INT_MAX/2&&c->q_heads<=INT_MAX/(2*c->head_dim)&&+ c->kv_heads<=INT_MAX/c->head_dim,"attention projection dimensions overflow");+ Q38_NEED(c->idx_qheads>0&&c->idx_kheads==1&&c->idx_dim>=c->rotary_dim,"indexer dimensions invalid");+ Q38_NEED(c->idx_ratio>0&&c->idx_budget>0&&c->idx_budget%c->idx_ratio==0&&+ c->idx_budget<=INT_MAX-c->idx_ratio+1,"indexer budget invalid");+ Q38_NEED(c->idx_qheads<INT_MAX&&c->idx_qheads<=INT_MAX/c->idx_dim-c->idx_kheads,+ "indexer projection dimensions overflow");+ Q38_NEED(c->experts>0&&c->experts<=Q38_MAX_EXPERTS,"experts=%d",c->experts);+ Q38_NEED(c->topk>0&&c->topk<=Q38_MAX_TOPK&&c->topk<=c->experts,"topk=%d",c->topk);+ Q38_NEED(c->inter<=INT_MAX/2,"expert projection dimensions overflow");+ Q38_NEED(c->inter>0&&c->shared_inter>0,"MoE widths invalid");+ Q38_NEED(c->dn_vheads>0&&c->dn_kheads>0&&c->dn_vheads%c->dn_kheads==0,"DeltaNet heads invalid");+ Q38_NEED(c->dn_kdim>0&&c->dn_vdim>0&&c->dn_vdim<=512&&c->dn_convk>=2&&c->dn_conv_dim>0&&+ c->dn_vdim<=INT_MAX/c->dn_vheads&&c->dn_kdim<=INT_MAX/c->dn_vdim,+ "DeltaNet dimensions invalid");+ Q38_NEED((uint64_t)c->dn_vheads<=SIZE_MAX/sizeof(float)/(uint64_t)c->dn_kdim/(uint64_t)c->dn_vdim&&+ (uint64_t)c->dn_conv_dim<=SIZE_MAX/sizeof(float)/(uint64_t)(c->dn_convk-1),+ "DeltaNet state dimensions overflow");+ Q38_NEED(c->ngram_size==3&&c->ngram_heads>0&&c->ngram_heads<=64&&+ c->ple_dim>0&&c->ple_dim%c->ngram_heads==0&&+ c->ngram_head_dim>0&&c->ngram_head_dim<=512&&+ c->ple_convk>=2&&c->ple_convk<=INT_MAX/c->ngram_size,+ "PLE dimensions invalid");+ Q38_NEED((uint64_t)c->hc_width<=SIZE_MAX/sizeof(float)/(uint64_t)(c->ple_convk-1)/(uint64_t)c->ngram_size,+ "PLE state dimensions overflow");+ Q38_NEED(c->ngram_parts>0&&c->ngram_parts<=Q38_MAX_PLE_PARTS,"PLE parts=%d",c->ngram_parts);+ Q38_NEED(c->ple_layer>=0&&c->ple_layer<c->layers,"PLE layer=%d",c->ple_layer);+}++static float *q38_load_tensor(Model *m,const char *name,int64_t want) {+ st_tensor *t=st_find(&m->S,name);+ if(!t){fprintf(stderr,"missing %s\n",name);exit(1);}+ if(t->numel!=want){fprintf(stderr,"%s: %lld elements, expected %lld\n",name,(long long)t->numel,(long long)want);exit(1);}+ float *p=falloc(want); st_read_f32(&m->S,name,p,1); return p;+}++static int q38_env_bool(const char *name,int default_value) {+ const char *value=getenv(name);if(!value||!*value)return default_value;+ if(value[0]=='0'&&!value[1])return 0;+ if(value[0]=='1'&&!value[1])return 1;+ fprintf(stderr,"%s must be exactly 0 or 1\n",name);exit(1);+}++static Q38Weight q38_load_weight(Model *m,const char *name,int rows,int cols) {+ st_tensor *tensor=st_find(&m->S,name);Q38Weight weight={0};+ if(!tensor){fprintf(stderr,"missing %s\n",name);exit(1);}+ if(tensor->rank!=2||tensor->shape[0]!=rows||tensor->shape[1]!=cols||+ tensor->numel!=(int64_t)rows*cols){+ fprintf(stderr,"%s: invalid matrix shape, expected [%d,%d]\n",name,rows,cols);exit(1);+ }+ if(tensor->dtype==0&&m->native_bf16){+ q38_weight_reserve(&weight,Q38_WEIGHT_BF16,rows,cols);+ if(tensor->nbytes!=weight.elements*(int64_t)sizeof(uint16_t)){+ fprintf(stderr,"%s: invalid BF16 byte count\n",name);exit(1);+ }+ st_read_raw_cap(&m->S,name,weight.data,tensor->nbytes,1);+ } else {+ if(tensor->dtype<0||tensor->dtype>2){+ fprintf(stderr,"%s: unsupported resident dtype %s\n",name,st_dtype_name(tensor->dtype));exit(1);+ }+ q38_weight_reserve(&weight,Q38_WEIGHT_F32,rows,cols);+ st_read_f32(&m->S,name,(float*)weight.data,1);+ }+ m->resident_weight_bytes+=q38_weight_bytes(&weight);+ return weight;+}++static void q38_name(Model *m,char *out,size_t cap,int layer,const char *suffix) {+ snprintf(out,cap,"%s.layers.%d.%s",m->prefix,layer,suffix);+}++static void q38_load_gr(Model *m,GatedResidual *g,int layer,const char *kind,int inject) {+ Cfg *c=&m->c; char nm[320],base[180];+ if(layer>=0) snprintf(base,sizeof base,"layers.%d.%s",layer,kind); else snprintf(base,sizeof base,"hyper_connection_mixer");+ snprintf(nm,sizeof nm,"%s.%s.hc_norm.weight",m->prefix,base); g->norm=q38_load_tensor(m,nm,c->hc_width);+ snprintf(nm,sizeof nm,"%s.%s.input_mix_weight_down.weight",m->prefix,base); g->down=q38_load_weight(m,nm,c->hc_rank,c->hc_width);+ snprintf(nm,sizeof nm,"%s.%s.input_mix_weight_up.weight",m->prefix,base); g->up=q38_load_weight(m,nm,c->hc_width,c->hc_rank);+ if(inject){snprintf(nm,sizeof nm,"%s.%s.block_inject_weight.weight",m->prefix,base);g->inject=q38_load_weight(m,nm,c->hc_count,c->hc_width);}+}++static void q38_load_ple(Model *m,Layer *l) {+ Cfg *c=&m->c; int i=c->ple_layer; char nm[320];+ q38_name(m,nm,sizeof nm,i,"ple.key_proj.weight");l->ple_key=q38_load_weight(m,nm,c->hc_width,c->ple_dim);+ q38_name(m,nm,sizeof nm,i,"ple.value_proj.weight");l->ple_value=q38_load_weight(m,nm,c->hidden,c->ple_dim);+ #define PL(field,suf,n) q38_name(m,nm,sizeof nm,i,"ple." suf); l->field=q38_load_tensor(m,nm,(n))+ PL(ple_norm_key,"norm_key.weight",c->hc_width);PL(ple_norm_query,"norm_query.weight",c->hc_width);+ PL(ple_norm_conv,"norm_conv.weight",c->hc_width);PL(ple_conv,"conv1d.weight",(int64_t)c->hc_width*c->ple_convk);+ #undef PL+ const char *bufs[]={"layer_multipliers","ngram_heads_vocab_sizes","ngram_heads_offsets"};+ int64_t *dsts[]={m->ple_multipliers,m->ple_head_vocab,m->ple_head_offset};+ int counts[]={c->ngram_size,c->ngram_heads,c->ngram_heads};+ for(int b=0;b<3;b++){+ q38_name(m,nm,sizeof nm,i,"ple.ple_embedding."); strncat(nm,bufs[b],sizeof(nm)-strlen(nm)-1);+ st_tensor *t=st_find(&m->S,nm); if(!t||t->dtype!=6||t->numel!=counts[b]){fprintf(stderr,"invalid %s\n",nm);exit(1);}+ st_read_raw_cap(&m->S,nm,dsts[b],(int64_t)counts[b]*8,1);+ }+ for(int h=0;h<c->ngram_heads;h++)+ Q38_NEED(m->ple_head_vocab[h]>0&&m->ple_head_offset[h]>=0&&+ m->ple_head_vocab[h]<=INT64_MAX-m->ple_head_offset[h],+ "invalid PLE head range %d",h);+ m->ple_part_count=0; m->ple_part_start[0]=0;+ q38_name(m,nm,sizeof nm,i,"ple.ple_embedding.ngram_embedding.weight");+ if(st_has(&m->S,nm)){+ st_tensor *t=st_find(&m->S,nm);+ if(!t||t->rank!=2||t->shape[0]<=0||t->shape[1]!=c->ngram_head_dim){fprintf(stderr,"invalid PLE table %s\n",nm);exit(1);}+ snprintf(m->ple_part_names[0],sizeof m->ple_part_names[0],"%s",nm);+ m->ple_parts[0]=t; m->ple_part_count=1;+ m->ple_part_start[1]=m->ple_parts[0]->shape[0];+ } else {+ for(int p=0;p<c->ngram_parts;p++){+ q38_name(m,nm,sizeof nm,i,"ple.ple_embedding.ngram_embedding.");+ size_t z=strlen(nm); snprintf(nm+z,sizeof(nm)-z,"shard_%d.weight",p);+ st_tensor *t=st_find(&m->S,nm); if(!t){fprintf(stderr,"missing PLE shard %s\n",nm);exit(1);}+ if(t->rank!=2||t->shape[0]<=0||t->shape[1]!=c->ngram_head_dim||+ m->ple_part_start[p]>INT64_MAX-t->shape[0]){fprintf(stderr,"invalid PLE shard %s\n",nm);exit(1);}+ snprintf(m->ple_part_names[p],sizeof m->ple_part_names[p],"%s",nm);+ m->ple_parts[p]=t; m->ple_part_start[p+1]=m->ple_part_start[p]+t->shape[0];+ m->ple_part_count++;+ }+ }+ q38_name(m,nm,sizeof nm,i,"ple.ple_embedding.ngram_embedding.weight_scale");+ m->ple_weight_scale=1.f;+ if(st_has(&m->S,nm)){+ st_tensor *t=st_find(&m->S,nm); if(t->numel!=1){fprintf(stderr,"invalid %s\n",nm);exit(1);}+ st_read_f32(&m->S,nm,&m->ple_weight_scale,1);+ }+ int64_t need=0; for(int h=0;h<c->ngram_heads;h++) if(m->ple_head_offset[h]+m->ple_head_vocab[h]>need) need=m->ple_head_offset[h]+m->ple_head_vocab[h];+ Q38_NEED(m->ple_part_start[m->ple_part_count]>=need,"PLE table rows %lld < required %lld",(long long)m->ple_part_start[m->ple_part_count],(long long)need);+}++static void q38_alloc_state(Model *m) {+ Cfg *c=&m->c;+ m->DN_rec=(float**)calloc((size_t)c->layers,sizeof(float*));+ m->DN_conv=(float**)calloc((size_t)c->layers,sizeof(float*));+ m->K=(float**)calloc((size_t)c->layers,sizeof(float*));+ m->V=(float**)calloc((size_t)c->layers,sizeof(float*));+ m->IK=(float**)calloc((size_t)c->layers,sizeof(float*));+ if(!m->DN_rec||!m->DN_conv||!m->K||!m->V||!m->IK){fprintf(stderr,"OOM model state metadata\n");exit(1);}+ for(int i=m->range_begin;i<m->range_end;i++) if(!c->is_attn[i]) {+ m->DN_rec[i]=(float*)calloc((size_t)c->dn_vheads*c->dn_kdim*c->dn_vdim,sizeof(float));+ m->DN_conv[i]=(float*)calloc((size_t)c->dn_conv_dim*(c->dn_convk-1),sizeof(float));+ if(!m->DN_rec[i]||!m->DN_conv[i]){fprintf(stderr,"OOM DeltaNet state\n");exit(1);}+ }+ if(c->ple_layer>=m->range_begin&&c->ple_layer<m->range_end){+ m->ple_history=(int64_t*)calloc(2,sizeof(int64_t));+ m->PLE_conv_state=(float*)calloc((size_t)c->hc_width*(c->ple_convk-1)*c->ngram_size,sizeof(float));+ if(!m->ple_history||!m->PLE_conv_state){fprintf(stderr,"OOM PLE state\n");exit(1);}+ }+}++/* ---- vision ------------------------------------------------------------- */++/* La torre e' in F32 residente: 27 blocchi da 1152 sono ~0.6 GB, che accanto ai+ * 9.2 GiB dei pesi densi non cambia la classe di macchina. Gli esperti restano+ * su disco; la torre no, perche' si usa una volta per immagine e non per token. */+static const float *q38_vis_tensor(Model *m,const char *suffix,int64_t expect) {+ char nm[512];+ snprintf(nm,sizeof nm,"%s.visual.%s",m->prefix,suffix);+ if(!st_has(&m->S,nm)){+ snprintf(nm,sizeof nm,"model.visual.%s",suffix);+ if(!st_has(&m->S,nm)){fprintf(stderr,"vision tensor missing: %s\n",suffix);exit(1);}+ }+ st_tensor *t=st_find(&m->S,nm);+ if(expect>0&&t->numel!=expect){+ fprintf(stderr,"vision tensor %s has %lld values, expected %lld -- refusing\n",+ nm,(long long)t->numel,(long long)expect);exit(1);+ }+ float *buf=(float*)malloc((size_t)t->numel*sizeof(float));+ if(!buf){fprintf(stderr,"OOM loading %s\n",nm);exit(1);}+ st_read_f32(&m->S,nm,buf,t->numel);+ m->resident_weight_bytes+=(uint64_t)t->numel*sizeof(float);+ return buf;+}++static void q38_vis_linear(Model *m,Q38Linear *l,const char *stem,int out,int in) {+ char nm[512];+ snprintf(nm,sizeof nm,"%s.weight",stem); l->w=q38_vis_tensor(m,nm,(int64_t)out*in);+ snprintf(nm,sizeof nm,"%s.bias",stem); l->b=q38_vis_tensor(m,nm,out);+ l->out=out; l->in=in;+}++static void q38_vis_norm(Model *m,Q38Norm *n,const char *stem,int width) {+ char nm[512];+ snprintf(nm,sizeof nm,"%s.weight",stem); n->w=q38_vis_tensor(m,nm,width);+ snprintf(nm,sizeof nm,"%s.bias",stem); n->b=q38_vis_tensor(m,nm,width);+}++static void q38_load_vision(Model *m) {+ Cfg *c=&m->c;+ if(!c->vis_depth) return;+ char probe[512];+ snprintf(probe,sizeof probe,"%s.visual.pos_embed.weight",m->prefix);+ if(!st_has(&m->S,probe)&&!st_has(&m->S,"model.visual.pos_embed.weight")){+ /* config multimodale ma pesi assenti: e' un export solo-testo di un+ * checkpoint multimodale. Spegnere la torre e dirlo e' meglio che+ * rifiutare un modello che per il testo funziona benissimo. */+ fprintf(stderr,"[qwen38] vision_config present but no visual weights; text only\n");+ c->vis_depth=0; return;+ }+ Q38Vision *v=&m->vis;+ memset(v,0,sizeof *v);+ v->depth=c->vis_depth; v->hidden=c->vis_hidden; v->heads=c->vis_heads;+ v->head_dim=c->vis_hidden/c->vis_heads; v->inter=c->vis_inter;+ v->patch=c->vis_patch; v->merge=c->vis_merge; v->temporal=c->vis_temporal;+ v->in_ch=c->vis_in_ch; v->out_hidden=c->vis_out_hidden;+ v->num_pos=c->vis_num_pos; v->side=(int)(sqrt((double)c->vis_num_pos)+0.5);+ v->eps=1e-6f;+ if(v->side*v->side!=v->num_pos){+ fprintf(stderr,"[qwen38] num_position_embeddings=%d is not a square grid -- refusing\n",+ v->num_pos);exit(1);+ }+ int features=v->in_ch*v->temporal*v->patch*v->patch;+ q38_vis_linear(m,&v->patch_embed,"patch_embed.proj",v->hidden,features);+ v->pos_embed=q38_vis_tensor(m,"pos_embed.weight",(int64_t)v->num_pos*v->hidden);+ v->blocks=(Q38VBlock*)calloc((size_t)v->depth,sizeof(Q38VBlock));+ if(!v->blocks){fprintf(stderr,"OOM vision blocks\n");exit(1);}+ for(int i=0;i<v->depth;i++){+ char stem[160];+ snprintf(stem,sizeof stem,"blocks.%d.norm1",i); q38_vis_norm(m,&v->blocks[i].norm1,stem,v->hidden);+ snprintf(stem,sizeof stem,"blocks.%d.norm2",i); q38_vis_norm(m,&v->blocks[i].norm2,stem,v->hidden);+ snprintf(stem,sizeof stem,"blocks.%d.attn.qkv",i); q38_vis_linear(m,&v->blocks[i].qkv,stem,3*v->hidden,v->hidden);+ snprintf(stem,sizeof stem,"blocks.%d.attn.proj",i); q38_vis_linear(m,&v->blocks[i].proj,stem,v->hidden,v->hidden);+ snprintf(stem,sizeof stem,"blocks.%d.mlp.linear_fc1",i); q38_vis_linear(m,&v->blocks[i].fc1,stem,v->inter,v->hidden);+ snprintf(stem,sizeof stem,"blocks.%d.mlp.linear_fc2",i); q38_vis_linear(m,&v->blocks[i].fc2,stem,v->hidden,v->inter);+ }+ int wide=v->hidden*v->merge*v->merge;+ q38_vis_norm(m,&v->merger_norm,"merger.norm",v->hidden);+ q38_vis_linear(m,&v->merger_fc1,"merger.linear_fc1",wide,wide);+ q38_vis_linear(m,&v->merger_fc2,"merger.linear_fc2",v->out_hidden,wide);+ m->vis_ready=1;+ fprintf(stderr,"[qwen38] vision tower: %d blocks, hidden %d, %d heads, patch %d, merge %d\n",+ v->depth,v->hidden,v->heads,v->patch,v->merge);+}++/* Esegue la torre su un'immagine gia' preprocessata e prepara la mappa+ * posizione->riga. `ids`/`n` sono i token del prompt: le righe vengono+ * assegnate ai token immagine nell'ordine in cui compaiono. */+static int q38_vision_attach(Model *m,const float *patches,int grid_h,int grid_w,+ const int *ids,int n) {+ Cfg *c=&m->c;+ if(!m->vis_ready) return -1;+ int tokens=(grid_h*grid_w)/(c->vis_merge*c->vis_merge);+ int slots=0;+ for(int i=0;i<n;i++) if(ids[i]==c->image_token) slots++;+ if(slots!=tokens){+ /* Il numero di segnaposto nel prompt DEVE essere quello che la griglia+ * produce. Se non lo e', il template e il preprocessing hanno visto due+ * immagini diverse, e proseguire vorrebbe dire mettere i vettori giusti+ * nelle posizioni sbagliate. */+ fprintf(stderr,"[qwen38] prompt has %d image placeholders but the grid gives %d tokens\n",+ slots,tokens);+ return -1;+ }+ free(m->vis_rows); free(m->vis_map);+ m->vis_rows=(float*)calloc((size_t)tokens*c->hidden,sizeof(float));+ m->vis_map=(int*)malloc((size_t)n*sizeof(int));+ if(!m->vis_rows||!m->vis_map){free(m->vis_rows);free(m->vis_map);+ m->vis_rows=NULL;m->vis_map=NULL;return -1;}+ if(q38_vision_forward(&m->vis,patches,grid_h,grid_w,m->vis_rows)!=tokens){+ free(m->vis_rows);free(m->vis_map);m->vis_rows=NULL;m->vis_map=NULL;return -1;}+ int next=0;+ for(int i=0;i<n;i++) m->vis_map[i]=(ids[i]==c->image_token)?next++:-1;+ m->vis_map_len=n; m->vis_rows_n=tokens;+ return tokens;+}++static void q38_vision_detach(Model *m) {+ free(m->vis_rows); free(m->vis_map);+ m->vis_rows=NULL; m->vis_map=NULL; m->vis_map_len=0; m->vis_rows_n=0;+}++static void model_init_range(Model *m,const char *snap,int cap,int bits,+ int layer_begin,int layer_end,int load_boundaries,+ int allocate_state) {+ (void)bits; memset(m,0,sizeof(*m)); double t0=now_s();+ m->native_fp8=q38_env_bool("Q38_NATIVE_FP8",1);+ m->native_bf16=q38_env_bool("Q38_NATIVE_BF16",1);+ m->expert_prefetch=q38_env_bool("Q38_EXPERT_PREFETCH",1);+ m->expert_parallel_reads=q38_env_bool("Q38_EXPERT_PARALLEL_READS",1);+ m->prefill_batch=q38_env_bool("Q38_PREFILL_BATCH",1);+ q38_load_cfg(&m->c,snap); q38_validate_cfg(&m->c); st_init(&m->S,snap);+ Cfg *c=&m->c; char nm[320];+ if(st_has(&m->S,"model.language_model.embed_tokens.weight")) snprintf(m->prefix,sizeof m->prefix,"model.language_model");+ else if(st_has(&m->S,"model.embed_tokens.weight")) snprintf(m->prefix,sizeof m->prefix,"model");+ else {fprintf(stderr,"checkpoint has no Qwen4-Exp text embedding\n");exit(1);}+ if(layer_end==0) layer_end=c->layers;+ if(layer_begin<0||layer_begin>=layer_end||layer_end>c->layers){fprintf(stderr,"invalid Qwen3.8 layer range [%d,%d)\n",layer_begin,layer_end);exit(1);}+ m->range_begin=layer_begin; m->range_end=layer_end;+ if(load_boundaries){+ snprintf(nm,sizeof nm,"%s.embed_tokens.weight",m->prefix); m->embed=q38_load_weight(m,nm,c->vocab,c->hidden);+ m->lm_head=q38_load_weight(m,"lm_head.weight",c->vocab,c->hidden);+ q38_load_gr(m,&m->final_gr,-1,NULL,0);+ }+ m->L=(Layer*)calloc((size_t)c->layers,sizeof(Layer));+ m->cache=(LCache*)calloc((size_t)c->layers,sizeof(LCache));+ m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers,+ sizeof(*m->expert_scales));+ if(!m->L||!m->cache||!m->expert_scales){fprintf(stderr,"OOM model metadata\n");exit(1);}+ for(int i=layer_begin;i<layer_end;i++){+ Layer *l=&m->L[i]; q38_load_gr(m,&l->attn_gr,i,"attn_hyper_connection",1); q38_load_gr(m,&l->mlp_gr,i,"mlp_hyper_connection",1);+ #define WLD(field,suf,o,in) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_weight(m,nm,(o),(in))+ #define VLD(field,suf,n) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_tensor(m,nm,(n))+ WLD(router,"mlp.gate.weight",c->experts,c->hidden);+ WLD(sh_g,"mlp.shared_expert.gate_proj.weight",c->shared_inter,c->hidden);+ WLD(sh_u,"mlp.shared_expert.up_proj.weight",c->shared_inter,c->hidden);+ WLD(sh_d,"mlp.shared_expert.down_proj.weight",c->hidden,c->shared_inter);+ VLD(sh_gate,"mlp.shared_expert_gate.weight",c->hidden);+ if(c->is_attn[i]){+ WLD(q,"self_attn.q_proj.weight",c->q_heads*c->head_dim*2,c->hidden);+ WLD(k,"self_attn.k_proj.weight",c->kv_heads*c->head_dim,c->hidden);+ WLD(v,"self_attn.v_proj.weight",c->kv_heads*c->head_dim,c->hidden);+ WLD(o,"self_attn.o_proj.weight",c->hidden,c->q_heads*c->head_dim);+ VLD(qn,"self_attn.q_norm.weight",c->head_dim);VLD(kn,"self_attn.k_norm.weight",c->head_dim);+ WLD(idx_qk,"self_attn.indexer.index_qk_proj.weight",(c->idx_qheads+c->idx_kheads)*c->idx_dim,c->hidden);+ VLD(idx_qn,"self_attn.indexer.q_layernorm.weight",c->idx_dim);VLD(idx_kn,"self_attn.indexer.k_layernorm.weight",c->idx_dim);+ } else {+ int vd=c->dn_vheads*c->dn_vdim;+ WLD(dn_qkv,"linear_attn.in_proj_qkv.weight",c->dn_conv_dim,c->hidden);+ WLD(dn_z,"linear_attn.in_proj_z.weight",vd,c->hidden);+ WLD(dn_b,"linear_attn.in_proj_b.weight",c->dn_vheads,c->hidden);+ WLD(dn_a,"linear_attn.in_proj_a.weight",c->dn_vheads,c->hidden);+ VLD(dn_conv,"linear_attn.conv1d.weight",(int64_t)c->dn_conv_dim*c->dn_convk);+ VLD(dn_dtbias,"linear_attn.dt_bias",c->dn_vheads);VLD(dn_alog,"linear_attn.A_log",c->dn_vheads);+ VLD(dn_norm,"linear_attn.norm.weight",c->dn_vdim);+ WLD(dn_out,"linear_attn.out_proj.weight",c->hidden,vd);+ }+ #undef WLD+ #undef VLD+ LCache *lc=&m->cache[i]; lc->cap=cap; lc->slots=(Slot*)calloc((size_t)cap,sizeof(Slot)); lc->by_expert=(int*)malloc((size_t)c->experts*sizeof(int));+ if(!lc->slots||!lc->by_expert){fprintf(stderr,"OOM expert cache\n");exit(1);} for(int e=0;e<c->experts;e++)lc->by_expert[e]=-1;+ }+ if(c->ple_layer>=layer_begin&&c->ple_layer<layer_end) q38_load_ple(m,&m->L[c->ple_layer]);+ /* La torre solo quando il motore possiede la sequenza intera: uno shard che+ * ospita solo alcuni layer non ha da fare niente con le immagini, e+ * caricarla la' sarebbe mezzo giga per nulla. */+ if(load_boundaries&&q38_env_bool("Q38_VISION",1)) q38_load_vision(m);+ if(allocate_state) q38_alloc_state(m);+ m->dense_load_s=now_s()-t0;+ fprintf(stderr,"[qwen38] native text weights: prefix=%s, %d layers, PLE=%d, cache=%d/layer, "+ "FP8=%s, BF16=%s (resident matrices %.2f GiB)\n",m->prefix,c->layers,c->ple_layer,cap,+ m->native_fp8?"native":"expanded-f32",m->native_bf16?"native":"expanded-f32",+ m->resident_weight_bytes/1073741824.0);+}++static void model_init(Model *m,const char *snap,int cap,int bits) {+ model_init_range(m,snap,cap,bits,0,0,1,1);+}++static void q38_gr_read(Model *m,const GatedResidual *g,const float *hyper,+ int S,float *mixed,float *inject) {+ const Cfg *c=&m->c;+ int H=c->hidden,W=c->hc_width,C=c->hc_count,R=c->hc_rank;+ if(C<=0||H<=0||W!=C*H){fprintf(stderr,"invalid gated-residual width\n");exit(1);}+ float *norm=falloc((int64_t)S*W),*low=falloc((int64_t)S*R),*mix=falloc((int64_t)S*W);+ /* Makes the initialized-input invariant visible to aggressive interprocedural+ * warning analysis; every element is overwritten by q38_rms0 below. */+ memset(norm,0,(size_t)S*W*sizeof(float));+ for(int s=0;s<S;s++) for(int b=0;b<C;b++)+ q38_rms0(norm+(int64_t)s*W+(int64_t)b*H,hyper+(int64_t)s*W+(int64_t)b*H,g->norm+(int64_t)b*H,H,c->eps);+ q38_dense_matmul(m,low,norm,&g->down,S,W,R);+ for(int64_t z=0;z<(int64_t)S*R;z++) low[z]=q38_silu(low[z]/C);+ q38_dense_matmul(m,mix,low,&g->up,S,R,W);+ for(int s=0;s<S;s++) for(int d=0;d<H;d++){+ float v=0.f;+ for(int b=0;b<C;b++) v+=q38_sigmoid(mix[(int64_t)s*W+(int64_t)b*H+d])*norm[(int64_t)s*W+(int64_t)b*H+d];+ mixed[(int64_t)s*H+d]=v/C;+ }+ if(inject){+ q38_dense_matmul(m,inject,norm,&g->inject,S,W,C);+ for(int64_t z=0;z<(int64_t)S*C;z++) inject[z]=2.f*q38_sigmoid(inject[z]/C);+ }+ free(norm);free(low);free(mix);+}++static void q38_gr_apply(const Cfg *c,float *hyper,const float *block,const float *inject,int S) {+ int H=c->hidden,C=c->hc_count,W=c->hc_width;+ for(int s=0;s<S;s++)for(int b=0;b<C;b++){+ float a=inject[(int64_t)s*C+b];+ for(int d=0;d<H;d++)hyper[(int64_t)s*W+(int64_t)b*H+d]+=a*block[(int64_t)s*H+d];+ }+}++typedef struct {+ st_tensor *tensor;+ int expert, projection;+} Q38ScaleDesc;++static int q38_scale_desc_cmp(const void *left,const void *right) {+ const Q38ScaleDesc *a=(const Q38ScaleDesc*)left;+ const Q38ScaleDesc *b=(const Q38ScaleDesc*)right;+ if(a->tensor->off<b->tensor->off)return -1;+ if(a->tensor->off>b->tensor->off)return 1;+ return 0;+}++static int q38_scale_group_span(Q38ScaleDesc *desc,int count,int *fd,+ int64_t *begin,int64_t *nbytes) {+ if(!desc||count<1||!fd||!begin||!nbytes)return -1;+ qsort(desc,(size_t)count,sizeof(*desc),q38_scale_desc_cmp);+ int group_fd=desc[0].tensor->fd;+ int64_t first=desc[0].tensor->off,cursor=first;+ for(int i=0;i<count;i++){+ st_tensor *tensor=desc[i].tensor;+ if(tensor->fd!=group_fd||tensor->off!=cursor||tensor->nbytes<=0||+ tensor->nbytes>INT64_MAX-cursor)return -1;+ cursor+=tensor->nbytes;+ }+ *fd=group_fd;*begin=first;*nbytes=cursor-first;return 0;+}++static void q38_decode_scale_tensor(float *out,const unsigned char *raw,+ const st_tensor *tensor) {+ if(tensor->dtype==2)memcpy(out,raw,(size_t)tensor->nbytes);+ else for(int64_t i=0;i<tensor->numel;i++){+ uint16_t half;memcpy(&half,raw+(size_t)i*sizeof(half),sizeof(half));+ out[i]=tensor->dtype==0?bf16_to_f32(half):f16_to_f32(half);+ }+}++/* The official checkpoint stores every layer's gate/up scale sidecars as one+ * compact range and every down sidecar as another. Normalize both ranges to+ * an expert-indexed F32 bank once, scattering by the numeric expert id rather+ * than the checkpoint's lexical tensor order (0, 1, 10, ...). Variants that+ * do not satisfy the compact-range invariant simply keep the established+ * per-matrix loader. */+static int q38_prepare_expert_scale_bank(Model *m,int layer) {+ Cfg *c=&m->c;+ if(!m->native_fp8||layer<0||layer>=c->layers)return 0;+ if(!m->expert_scales){+ m->expert_scales=(Q38ExpertScaleCache*)calloc((size_t)c->layers,+ sizeof(*m->expert_scales));+ if(!m->expert_scales){fprintf(stderr,"OOM expert scale metadata\n");exit(1);}+ }+ Q38ExpertScaleCache *cache=&m->expert_scales[layer];+ if(cache->ready)return cache->ready>0;+ int64_t scale_count=fp8_nblk(c->inter)*fp8_nblk(c->hidden);+ if(scale_count<1||(uint64_t)c->experts>SIZE_MAX/(3u*(uint64_t)scale_count*sizeof(float))){+ fprintf(stderr,"invalid expert scale-bank geometry\n");exit(1);+ }+ Q38ScaleDesc *gate_up=(Q38ScaleDesc*)malloc((size_t)c->experts*2*sizeof(*gate_up));+ Q38ScaleDesc *down=(Q38ScaleDesc*)malloc((size_t)c->experts*sizeof(*down));+ if(!gate_up||!down){fprintf(stderr,"OOM expert scale descriptors\n");exit(1);}+ int ngu=0,ndown=0;char suffix[192],name[320];+ const char *projection[3]={"gate_proj","up_proj","down_proj"};+ for(int expert=0;expert<c->experts;expert++)for(int kind=0;kind<3;kind++){+ int rows=kind==2?c->hidden:c->inter;+ int cols=kind==2?c->inter:c->hidden;+ int length=snprintf(suffix,sizeof suffix,+ "mlp.experts.%d.%s.weight_scale_inv",expert,projection[kind]);+ if(length<0||(size_t)length>=sizeof suffix)goto incompatible;+ q38_name(m,name,sizeof name,layer,suffix);+ st_tensor *tensor=st_find(&m->S,name);+ int64_t block_rows=fp8_nblk(rows),block_cols=fp8_nblk(cols);+ if(!tensor||tensor->dtype<0||tensor->dtype>2||tensor->rank!=2||+ tensor->shape[0]!=block_rows||tensor->shape[1]!=block_cols||+ tensor->numel!=scale_count||+ tensor->numel>INT64_MAX/st_dtype_esz(tensor->dtype)||+ tensor->nbytes!=tensor->numel*st_dtype_esz(tensor->dtype))+ goto incompatible;+ Q38ScaleDesc item={tensor,expert,kind};+ if(kind<2)gate_up[ngu++]=item;else down[ndown++]=item;+ }+ int gu_fd,down_fd;int64_t gu_begin,down_begin,gu_bytes,down_bytes;+ if(q38_scale_group_span(gate_up,ngu,&gu_fd,&gu_begin,&gu_bytes)||+ q38_scale_group_span(down,ndown,&down_fd,&down_begin,&down_bytes)||+ (uint64_t)gu_bytes>SIZE_MAX||(uint64_t)down_bytes>SIZE_MAX)+ goto incompatible;+ float *values=(float*)malloc((size_t)c->experts*3*(size_t)scale_count*sizeof(float));+ unsigned char *gu_raw=(unsigned char*)malloc((size_t)gu_bytes);+ unsigned char *down_raw=(unsigned char*)malloc((size_t)down_bytes);+ if(!values||!gu_raw||!down_raw){fprintf(stderr,"OOM resident expert scales\n");exit(1);}+ double started=now_s();+ st_read_range_raw_cap(&m->S,gu_fd,gu_begin,gu_bytes,gu_raw,gu_bytes,1,+ "pread Qwen3.8 gate/up scales");+ st_read_range_raw_cap(&m->S,down_fd,down_begin,down_bytes,down_raw,down_bytes,1,+ "pread Qwen3.8 down scales");+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_scale_reads+=2;+ for(int i=0;i<ngu;i++){+ Q38ScaleDesc *item=&gate_up[i];+ float *dst=values+((int64_t)item->expert*3+item->projection)*scale_count;+ q38_decode_scale_tensor(dst,gu_raw+(item->tensor->off-gu_begin),item->tensor);+ }+ for(int i=0;i<ndown;i++){+ Q38ScaleDesc *item=&down[i];+ float *dst=values+((int64_t)item->expert*3+item->projection)*scale_count;+ q38_decode_scale_tensor(dst,down_raw+(item->tensor->off-down_begin),item->tensor);+ }+ free(gu_raw);free(down_raw);free(gate_up);free(down);+ cache->values=values;cache->scale_count=scale_count;cache->ready=1;+ m->expert_scale_bytes+=(uint64_t)c->experts*3*(uint64_t)scale_count*sizeof(float);+ return 1;+incompatible:+ free(gate_up);free(down);cache->ready=-1;return 0;+}++static void q38_bind_borrowed_fp8(Q38Weight *weight,void *data,float *scales,+ int rows,int cols) {+ q38_weight_free(weight);+ weight->data=data;weight->scales=scales;weight->rows=rows;weight->cols=cols;+ weight->elements=(int64_t)rows*cols;+ weight->scale_count=fp8_nblk(rows)*fp8_nblk(cols);+ weight->kind=Q38_WEIGHT_FP8;+}++static void q38_bind_fp8_slot(Slot *slot,float *scales,int scale_count,+ int hidden,int intermediate) {+ int64_t matrix_bytes=(int64_t)hidden*intermediate;+ if(matrix_bytes<1||matrix_bytes>INT64_MAX/3||+ (uint64_t)(matrix_bytes*3)>SIZE_MAX||+ scale_count!=fp8_nblk(hidden)*fp8_nblk(intermediate)){+ fprintf(stderr,"invalid native FP8 expert slab geometry\n");exit(1);+ }+ q38_weight_free(&slot->gate);q38_weight_free(&slot->up);+ q38_weight_free(&slot->down);+ int64_t slab_bytes=matrix_bytes*3;+ if(!slot->fp8_slab||slot->fp8_slab_bytes!=slab_bytes){+ void *replacement=realloc(slot->fp8_slab,(size_t)slab_bytes);+ if(!replacement){fprintf(stderr,"OOM native FP8 expert slab\n");exit(1);}+ slot->fp8_slab=replacement;slot->fp8_slab_bytes=slab_bytes;+ }+ unsigned char *raw=(unsigned char*)slot->fp8_slab;+ q38_bind_borrowed_fp8(&slot->gate,raw,scales,intermediate,hidden);+ q38_bind_borrowed_fp8(&slot->up,raw+matrix_bytes,scales+scale_count,+ intermediate,hidden);+ q38_bind_borrowed_fp8(&slot->down,raw+2*matrix_bytes,scales+2*scale_count,+ hidden,intermediate);+}++static int q38_native_fp8_expert_tensors(Model *m,int layer,int expert,+ st_tensor *weight[3]) {+ if(!m->native_fp8)return 0;+ Cfg *c=&m->c;const char *projection[3]={"gate_proj","up_proj","down_proj"};+ int rows[3]={c->inter,c->inter,c->hidden};+ int cols[3]={c->hidden,c->hidden,c->inter};+ char suffix[192],name[320];+ for(int kind=0;kind<3;kind++){+ int length=snprintf(suffix,sizeof suffix,"mlp.experts.%d.%s.weight",+ expert,projection[kind]);+ if(length<0||(size_t)length>=sizeof suffix)return 0;+ q38_name(m,name,sizeof name,layer,suffix);weight[kind]=st_find(&m->S,name);+ if(!weight[kind]||weight[kind]->dtype!=4||weight[kind]->rank!=2||+ weight[kind]->shape[0]!=rows[kind]||weight[kind]->shape[1]!=cols[kind]||+ weight[kind]->nbytes!=(int64_t)rows[kind]*cols[kind])return 0;+ }+ if(weight[0]->fd!=weight[1]->fd||+ weight[0]->off>INT64_MAX-weight[0]->nbytes||+ weight[1]->off!=weight[0]->off+weight[0]->nbytes)return 0;+ return 1;+}++static void q38_load_native_fp8_ranges(Model *m,int layer,int expert,Slot *slot,+ st_tensor *weight[3]) {+ Cfg *c=&m->c;+ Q38ExpertScaleCache *cache=&m->expert_scales[layer];+ float *scales=cache->values+(int64_t)expert*3*cache->scale_count;+ /* Se i tre intervalli sono mappati (COLI_MAP_EXPERTS=1), lo slot li PUNTA+ * invece di copiarli: niente slab, niente 14 MB per miss. */+ {+ const uint8_t *pg=(const uint8_t*)st_map_shard_range(weight[0]->fd,weight[0]->off,weight[0]->nbytes);+ const uint8_t *pu=(const uint8_t*)st_map_shard_range(weight[1]->fd,weight[1]->off,weight[1]->nbytes);+ const uint8_t *pd=(const uint8_t*)st_map_shard_range(weight[2]->fd,weight[2]->off,weight[2]->nbytes);+ if(pg&&pu&&pd){+ int sc=(int)cache->scale_count;+ q38_bind_borrowed_fp8(&slot->gate,(void*)pg,scales,c->inter,c->hidden);+ q38_bind_borrowed_fp8(&slot->up,(void*)pu,scales+sc,c->inter,c->hidden);+ q38_bind_borrowed_fp8(&slot->down,(void*)pd,scales+2*sc,c->hidden,c->inter);+ if(slot->fp8_slab){free(slot->fp8_slab);slot->fp8_slab=NULL;slot->fp8_slab_bytes=0;}+ return;+ }+ }+ q38_bind_fp8_slot(slot,scales,(int)cache->scale_count,c->hidden,c->inter);+ int64_t pair_bytes=weight[0]->nbytes+weight[1]->nbytes;+ unsigned char *raw=(unsigned char*)slot->fp8_slab;+ st_read_range_raw_cap(&m->S,weight[0]->fd,weight[0]->off,pair_bytes,+ raw,pair_bytes,1,"pread Qwen3.8 gate/up expert");+ st_read_range_raw_cap(&m->S,weight[2]->fd,weight[2]->off,weight[2]->nbytes,+ raw+pair_bytes,slot->fp8_slab_bytes-pair_bytes,1,+ "pread Qwen3.8 down expert");+}++static int q38_try_load_native_fp8_expert(Model *m,int layer,int expert,Slot *slot) {+ st_tensor *weight[3];+ if(!q38_native_fp8_expert_tensors(m,layer,expert,weight)||+ !q38_prepare_expert_scale_bank(m,layer))return 0;+ double started=now_s();+ q38_load_native_fp8_ranges(m,layer,expert,slot,weight);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ m->expert_weight_reads+=2;m->expert_pair_reads++;+ return 1;+}++static void q38_prefetch_native_fp8_experts(Model *m,int layer,+ const int *experts,int count) {+ if(!m->expert_prefetch||!m->native_fp8||!experts||count<1||+ !q38_prepare_expert_scale_bank(m,layer))return;+ LCache *cache=&m->cache[layer];+ for(int index=0;index<count;index++){+ int expert=experts[index];st_tensor *weight[3];+ if(expert<0||expert>=m->c.experts||cache->by_expert[expert]>=0||+ !q38_native_fp8_expert_tensors(m,layer,expert,weight))continue;+ posix_fadvise(weight[0]->fd,weight[0]->off,+ weight[0]->nbytes+weight[1]->nbytes,POSIX_FADV_WILLNEED);+ posix_fadvise(weight[2]->fd,weight[2]->off,weight[2]->nbytes,+ POSIX_FADV_WILLNEED);+ m->expert_prefetch_ranges+=2;+ }+}++static void q38_load_fp8_expert_weight(Model *m,const char *wn,const char *sn,+ Q38Weight *out,int O,int I) {+ st_tensor *w=st_find(&m->S,wn),*sc=st_find(&m->S,sn);+ int nb_o=(O+127)/128,nb_i=(I+127)/128;+ if(!w||w->dtype!=4||w->rank!=2||w->shape[0]!=O||w->shape[1]!=I||+ w->nbytes!=(int64_t)O*I||!sc||sc->dtype<0||sc->dtype>2||+ sc->rank!=2||sc->shape[0]!=nb_o||sc->shape[1]!=nb_i||+ sc->numel!=(int64_t)nb_o*nb_i){+ fprintf(stderr,"invalid block-FP8 expert matrix %s / %s\n",wn,sn);exit(1);+ }+ if(m->native_fp8){+ q38_weight_reserve(out,Q38_WEIGHT_FP8,O,I);+ double started=now_s();+ st_read_raw_cap(&m->S,wn,out->data,w->nbytes,1);+ st_read_f32(&m->S,sn,out->scales,1);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ m->expert_weight_reads++;m->expert_scale_reads++;+ return;+ }+ q38_weight_reserve(out,Q38_WEIGHT_F32,O,I);+ uint8_t *raw=(uint8_t*)malloc((size_t)w->nbytes);float *scale=falloc(sc->numel);+ if(!raw){fprintf(stderr,"OOM FP8 expert staging\n");exit(1);}+ double started=now_s();+ st_read_raw_cap(&m->S,wn,raw,w->nbytes,1);st_read_f32(&m->S,sn,scale,1);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ m->expert_weight_reads++;m->expert_scale_reads++;started=now_s();+ float *decoded=(float*)out->data;+ #pragma omp parallel for schedule(static)+ for(int o=0;o<O;o++)for(int i=0;i<I;i++)+ decoded[(int64_t)o*I+i]=e4m3_decode(raw[(int64_t)o*I+i])*+ scale[(o/128)*nb_i+i/128];+ q38_tm_add(m,Q38_TM_FP8_EXPAND,started);+ free(raw);free(scale);+}++static void q38_load_expert_weight(Model *m,const char *name,Q38Weight *out,+ int O,int I) {+ st_tensor *tensor=st_find(&m->S,name);+ if(!tensor||tensor->rank!=2||tensor->shape[0]!=O||tensor->shape[1]!=I||+ tensor->numel!=(int64_t)O*I||tensor->dtype<0||tensor->dtype>2){+ fprintf(stderr,"invalid expert matrix %s\n",name);exit(1);+ }+ Q38WeightKind kind=m->native_bf16&&tensor->dtype==0?Q38_WEIGHT_BF16:Q38_WEIGHT_F32;+ q38_weight_reserve(out,kind,O,I);double started=now_s();+ if(kind==Q38_WEIGHT_BF16)+ st_read_raw_cap(&m->S,name,out->data,tensor->nbytes,1);+ else st_read_f32(&m->S,name,(float*)out->data,1);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;+}++static void q38_load_expert_slice(Model *m,const char *name,const st_tensor *tensor,+ int64_t element_offset,Q38Weight *out,int O,int I) {+ if(!tensor||tensor->dtype<0||tensor->dtype>2||element_offset<0||+ element_offset>tensor->numel-(int64_t)O*I){+ fprintf(stderr,"invalid expert slice %s\n",name);exit(1);+ }+ Q38WeightKind kind=m->native_bf16&&tensor->dtype==0?Q38_WEIGHT_BF16:Q38_WEIGHT_F32;+ q38_weight_reserve(out,kind,O,I);double started=now_s();+ if(kind==Q38_WEIGHT_BF16){+ int64_t bytes=(int64_t)O*I*(int64_t)sizeof(uint16_t);+ st_read_slice_raw_cap(&m->S,name,element_offset*(int64_t)sizeof(uint16_t),+ bytes,out->data,bytes,1);+ } else st_read_slice_f32(&m->S,name,element_offset,(int64_t)O*I,(float*)out->data,1);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;+}++static void q38_load_expert(Model *m,int layer,int eid,Slot *s) {+ Cfg *c=&m->c; int H=c->hidden,I=c->inter; char nm[320],sn[340];+ q38_name(m,nm,sizeof nm,layer,"mlp.experts.gate_up_proj");+ if(st_has(&m->S,nm)){+ st_tensor *t=st_find(&m->S,nm);+ if(t->rank!=3||t->shape[0]!=c->experts||t->shape[1]!=2*I||t->shape[2]!=H){fprintf(stderr,"invalid %s\n",nm);exit(1);}+ int64_t base=(int64_t)eid*2*I*H;+ q38_load_expert_slice(m,nm,t,base,&s->gate,I,H);+ q38_load_expert_slice(m,nm,t,base+(int64_t)I*H,&s->up,I,H);+ q38_name(m,nm,sizeof nm,layer,"mlp.experts.down_proj"); t=st_find(&m->S,nm);+ if(!t||t->rank!=3||t->shape[0]!=c->experts||t->shape[1]!=H||t->shape[2]!=I){fprintf(stderr,"invalid %s\n",nm);exit(1);}+ q38_load_expert_slice(m,nm,t,(int64_t)eid*H*I,&s->down,H,I);+ return;+ }+ if(q38_try_load_native_fp8_expert(m,layer,eid,s))return;+ const char *kind[3]={"gate_proj","up_proj","down_proj"};Q38Weight *dst[3]={&s->gate,&s->up,&s->down};+ int os[3]={I,I,H},is[3]={H,H,I};+ for(int k=0;k<3;k++){+ char suf[192]; snprintf(suf,sizeof suf,"mlp.experts.%d.%s.weight",eid,kind[k]);q38_name(m,nm,sizeof nm,layer,suf);+ st_tensor *t=st_find(&m->S,nm); if(!t){fprintf(stderr,"missing %s\n",nm);exit(1);}+ if(t->dtype==4){snprintf(sn,sizeof sn,"%s_scale_inv",nm);q38_load_fp8_expert_weight(m,nm,sn,dst[k],os[k],is[k]);}+ else q38_load_expert_weight(m,nm,dst[k],os[k],is[k]);+ }+}++/* One byte per expert: routed in this turn or not. The dashboard's Brain tab+ * reads it as the HITS bitmap after every turn (serve_hits in qwen38.c),+ * cleared there. Marked on both lookup paths, single and batched. */+static pthread_mutex_t g_q38_ehit_mx=PTHREAD_MUTEX_INITIALIZER;+static void q38_ehit_mark(Model *m,int layer,int eid) {+ const Cfg *c=&m->c;+ /* The first touch can come from a parallel region (qwen36: the tier+ * warmstart's omp loop calls expert_get from twelve threads at once):+ * one thread published m->ehit while it was still filling the rows and+ * a sibling dereferenced m->ehit[layer] == NULL -- SIGSEGV in about one+ * run in twelve on a CUDA warmstart. Build the table privately, publish+ * it once under a lock (double-checked), and read it with acquire order. */+ uint8_t **ehit=__atomic_load_n(&m->ehit,__ATOMIC_ACQUIRE);+ if(!ehit){+ pthread_mutex_lock(&g_q38_ehit_mx);+ ehit=m->ehit;+ if(!ehit){+ ehit=(uint8_t**)calloc((size_t)c->layers,sizeof(uint8_t*));+ for(int i=0;i<c->layers;i++)ehit[i]=(uint8_t*)calloc((size_t)c->experts,1);+ __atomic_store_n(&m->ehit,ehit,__ATOMIC_RELEASE);+ }+ pthread_mutex_unlock(&g_q38_ehit_mx);+ }+ if(layer>=0&&layer<c->layers&&eid>=0&&eid<c->experts)ehit[layer][eid]=1;+}+static Slot *q38_expert_get(Model *m,int layer,int eid) {+ q38_ehit_mark(m,layer,eid);+ LCache *lc=&m->cache[layer]; int si=lc->by_expert[eid];+ if(si>=0){m->hits++;lc->slots[si].used=++m->clock;return &lc->slots[si];}+ m->miss++; Slot *s;+ if(lc->n<lc->cap){s=&lc->slots[lc->n++];s->eid=-1;}+ else {+ int victim=0;for(int i=1;i<lc->n;i++)if(lc->slots[i].used<lc->slots[victim].used)victim=i;+ s=&lc->slots[victim];if(s->eid>=0)lc->by_expert[s->eid]=-1;+ }+ s->eid=-1;q38_load_expert(m,layer,eid,s);s->eid=eid;s->used=++m->clock;lc->by_expert[eid]=(int)(s-lc->slots);return s;+}+typedef struct {int expert;Slot *slot;@@ -1341,12 +1341,12 @@ static int q38_expert_batch_fallback(Model *m,Q38ExpertBatchFallback reason,fprintf(stderr,"; using serial expert reads\n");return 0;}--/* Reserve an entire routed demand set on the main thread, fill distinct slots- * in parallel, then publish every cache index on the main thread. Demand-set- * residents are protected from victim selection, so no worker can overwrite a- * slot another selected expert will consume. Smaller caches and heterogeneous- * layouts retain the serial LRU path. */++/* Reserve an entire routed demand set on the main thread, fill distinct slots+ * in parallel, then publish every cache index on the main thread. Demand-set+ * residents are protected from victim selection, so no worker can overwrite a+ * slot another selected expert will consume. Smaller caches and heterogeneous+ * layouts retain the serial LRU path. */static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,Slot **selected) {if(!experts||!selected||count<2)return 0;@@ -1363,1026 +1363,1026 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,if(!q38_prepare_expert_scale_bank(m,layer))return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK,layer,0,0);- Q38ExpertLoadJob jobs[Q38_MAX_TOPK];- for(int index=0;index<count;index++){- int expert=experts[index];- if(expert<0||expert>=m->c.experts)return 0;+ Q38ExpertLoadJob jobs[Q38_MAX_TOPK];+ for(int index=0;index<count;index++){+ int expert=experts[index];+ if(expert<0||expert>=m->c.experts)return 0;q38_ehit_mark(m,layer,expert);for(int previous=0;previous<index;previous++)if(experts[previous]==expert)return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_DUPLICATE,layer,expert,0);- int slot_index=cache->by_expert[expert];- if(slot_index>=0){- if(slot_index>=cache->n||cache->slots[slot_index].eid!=expert)return 0;- continue;+ int slot_index=cache->by_expert[expert];+ if(slot_index>=0){+ if(slot_index>=cache->n||cache->slots[slot_index].eid!=expert)return 0;+ continue;}st_tensor *weight[3];if(!q38_native_fp8_expert_tensors(m,layer,expert,weight))return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_LAYOUT,layer,expert,0);- }- unsigned char *protected_slots=(unsigned char*)calloc((size_t)cache->cap,1);- if(!protected_slots){fprintf(stderr,"OOM expert batch reservations\n");exit(1);}- for(int index=0;index<count;index++){- int slot_index=cache->by_expert[experts[index]];- if(slot_index>=0)protected_slots[slot_index]=1;- }- int job_count=0;- for(int index=0;index<count;index++){- int expert=experts[index],slot_index=cache->by_expert[expert];Slot *slot;- if(slot_index>=0){- slot=&cache->slots[slot_index];m->hits++;- }else{- m->miss++;- if(cache->n<cache->cap){- slot=&cache->slots[cache->n++];slot->eid=-1;- }else{- int victim=-1;- for(int candidate=0;candidate<cache->n;candidate++)- if(!protected_slots[candidate]&&- (victim<0||cache->slots[candidate].used<cache->slots[victim].used))- victim=candidate;- if(victim<0){- fprintf(stderr,"Qwen3.8 expert demand set has no reservable cache slot\n");- exit(1);- }- slot=&cache->slots[victim];- if(slot->eid>=0)cache->by_expert[slot->eid]=-1;- }- slot->eid=-1;jobs[job_count].expert=expert;jobs[job_count].slot=slot;- if(!q38_native_fp8_expert_tensors(m,layer,expert,jobs[job_count].weight)){- fprintf(stderr,"Qwen3.8 expert layout changed during batch reservation\n");exit(1);- }- job_count++;- }- slot->used=++m->clock;selected[index]=slot;- protected_slots[slot-cache->slots]=1;- }- free(protected_slots);- if(job_count){- int workers=job_count;-#ifdef _OPENMP- int thread_limit=omp_get_max_threads();if(workers>thread_limit)workers=thread_limit;-#else- (void)workers; /* only the pragma below reads it, and that is gone without OpenMP */-#endif- double started=now_s();- #pragma omp parallel for schedule(static) num_threads(workers) if(job_count>1)- for(int job=0;job<job_count;job++)- q38_load_native_fp8_ranges(m,layer,jobs[job].expert,jobs[job].slot,- jobs[job].weight);- q38_tm_add(m,Q38_TM_EXPERT_READ,started);- m->expert_weight_reads+=(uint64_t)job_count*2;- m->expert_pair_reads+=(uint64_t)job_count;- if(job_count>1)m->expert_parallel_batches++;- for(int job=0;job<job_count;job++){- Slot *slot=jobs[job].slot;slot->eid=jobs[job].expert;- cache->by_expert[jobs[job].expert]=(int)(slot-cache->slots);- }- }- return 1;-}--static void q38_ple_row(Model *m,int64_t row,float *out) {- Cfg *c=&m->c; int p=0;- while(p+1<m->ple_part_count&&row>=m->ple_part_start[p+1])p++;- if(p>=m->ple_part_count||row<m->ple_part_start[p]){fprintf(stderr,"PLE row out of range: %lld\n",(long long)row);exit(1);}- int64_t local=row-m->ple_part_start[p]; st_tensor *t=m->ple_parts[p]; const char *nm=m->ple_part_names[p];- Q38_NEED(c->ngram_head_dim>0&&local>=0&&local<=INT64_MAX/c->ngram_head_dim,- "PLE row byte offset overflows");- if(t->dtype==4){- uint8_t raw[512]; Q38_NEED(c->ngram_head_dim<=(int)sizeof raw,"PLE row too wide");- st_read_slice_raw_cap(&m->S,nm,local*c->ngram_head_dim,c->ngram_head_dim,raw,sizeof raw,1);- for(int d=0;d<c->ngram_head_dim;d++)out[d]=e4m3_decode(raw[d])*m->ple_weight_scale;- } else st_read_slice_f32(&m->S,nm,local*c->ngram_head_dim,c->ngram_head_dim,out,1);-}--static int64_t q38_hash_row(Model *m,int head,int ngram,int64_t cur,int64_t p1,int64_t p2) {- uint64_t x=(uint64_t)cur*(uint64_t)m->ple_multipliers[0];- x^=(uint64_t)p1*(uint64_t)m->ple_multipliers[1];- if(ngram==3)x^=(uint64_t)p2*(uint64_t)m->ple_multipliers[2];- int64_t sx=(int64_t)x,mod=m->ple_head_vocab[head],r=sx%mod;if(r<0)r+=mod;- return m->ple_head_offset[head]+r;-}---/* Anticipa le letture PLE all'inizio del forward invece di emetterle inline al- * layer che le usa.- *- * Perche' si puo' fare, e perche' NON si puo' fare con gli esperti: gli indici- * delle righe PLE sono funzione pura degli id dei token e della finestra di due- * token che li precede -- q38_hash_row non guarda nessuno stato nascosto. Sono- * quindi noti PRIMA che parta un solo calcolo. Gli esperti no: quelli dipendono- * dal router del layer precedente, e per questo la loro finestra di anticipo e'- * di un layer e non di tutto il forward.- *- * Il PLE cade sul layer 2 di 48, quindi fra l'emissione e l'uso ci sono due- * layer interi di calcolo -- con il loro streaming di esperti, che su questo- * motore e' la parte lenta. E' abbondantemente il tempo di far arrivare 16- * righe da 160 byte.- *- * Sono anche letture PARALLELE invece che in fila: prima erano 16 pread seriali- * a queue depth 1 per token, che in prefill diventano lunghezza-del-prompt per- * 16, una alla volta.- *- * Riordino puro: stessi byte, letti prima. I token non cambiano, e il test lo- * pretende invece di darlo per scontato. */-static void q38_ple_prefetch(Model *m,const int *ids,int S) {- Cfg *c=&m->c;- free(m->ple_pref); m->ple_pref=NULL; m->ple_pref_rows=0;- if(c->ple_layer<0||S<1||c->ngram_heads<1||c->ngram_head_dim<1) return;- if(!q38_env_bool("Q38_PLE_PREFETCH",1)) return;-- int64_t per_row=c->ngram_head_dim, per_pos=(int64_t)c->ngram_heads*per_row;- if(S>INT_MAX/c->ngram_heads) return;- float *buffer=(float*)malloc((size_t)S*per_pos*sizeof(float));- int64_t *rows=(int64_t*)malloc((size_t)S*c->ngram_heads*sizeof(int64_t));- if(!buffer||!rows){free(buffer);free(rows);return;} /* niente prefetch: si legge inline */-- /* La finestra di due token viene SIMULATA, non mutata: q38_ple la aggiorna- * per conto suo mentre gira, e toccarla qui la farebbe avanzare due volte. */- int64_t history[2]={m->ple_history[0],m->ple_history[1]};- int history_len=m->ple_history_len;- for(int s=0;s<S;s++){- int64_t p1=history_len>=1?history[history_len-1]:c->eos_id;- int64_t p2=history_len>=2?history[history_len-2]:c->eos_id;- for(int h=0;h<c->ngram_heads;h++){- int ng=h<c->heads_per_ngram?2:3;- rows[(int64_t)s*c->ngram_heads+h]=q38_hash_row(m,h,ng,ids[s],p1,p2);- }- if(ids[s]==c->eos_id) history_len=0;- else if(history_len==0){history[0]=ids[s];history_len=1;}- else if(history_len==1){history[1]=ids[s];history_len=2;}- else {history[0]=history[1];history[1]=ids[s];}- }-- int64_t total=(int64_t)S*c->ngram_heads;- volatile int failed=0;- #pragma omp parallel for schedule(static)- for(int64_t i=0;i<total;i++){- if(failed) continue;- q38_ple_row(m,rows[i],buffer+i*per_row);- }- free(rows);- if(failed){free(buffer);return;}- m->ple_pref=buffer; m->ple_pref_rows=S;-}--static void q38_ple(Model *m,const int *ids,int S,const float *hyper,float *out) {- double phase_started=now_s();- Cfg *c=&m->c; Layer *l=&m->L[c->ple_layer]; int H=c->hidden,C=c->hc_count,W=c->hc_width,E=c->ple_dim;- float *emb=falloc(E),*keys=falloc(W),*value=falloc(H),*kn=falloc(W),*qn=falloc(W),*gated=falloc(W),*norm=falloc(W);- int state_len=(c->ple_convk-1)*c->ngram_size; float *ring=m->PLE_conv_state;- for(int s=0;s<S;s++){- int64_t p1=m->ple_history_len>=1?m->ple_history[m->ple_history_len-1]:c->eos_id;- int64_t p2=m->ple_history_len>=2?m->ple_history[m->ple_history_len-2]:c->eos_id;- if(m->ple_pref&&s<m->ple_pref_rows){- /* gia' in memoria: le ha portate q38_ple_prefetch mentre i primi- * layer calcolavano */- memcpy(emb,m->ple_pref+(int64_t)s*c->ngram_heads*c->ngram_head_dim,- (size_t)c->ngram_heads*c->ngram_head_dim*sizeof(float));- } else for(int h=0;h<c->ngram_heads;h++){- int ng=h<c->heads_per_ngram?2:3; int64_t row=q38_hash_row(m,h,ng,ids[s],p1,p2);- q38_ple_row(m,row,emb+(int64_t)h*c->ngram_head_dim);- }- q38_dense_matmul(m,keys,emb,&l->ple_key,1,E,W);q38_dense_matmul(m,value,emb,&l->ple_value,1,E,H);- for(int b=0;b<C;b++){- q38_rms0(kn+(int64_t)b*H,keys+(int64_t)b*H,l->ple_norm_key+(int64_t)b*H,H,c->eps);- q38_rms0(qn+(int64_t)b*H,hyper+(int64_t)s*W+(int64_t)b*H,l->ple_norm_query+(int64_t)b*H,H,c->eps);- float dot=0.f;for(int d=0;d<H;d++)dot+=kn[(int64_t)b*H+d]*qn[(int64_t)b*H+d];dot/=sqrtf((float)H);- float shaped=copysignf(sqrtf(fmaxf(fabsf(dot),1e-6f)),dot),g=q38_sigmoid(shaped);- for(int d=0;d<H;d++)gated[(int64_t)b*H+d]=g*value[d];- q38_rms0(norm+(int64_t)b*H,gated+(int64_t)b*H,l->ple_norm_conv+(int64_t)b*H,H,c->eps);- }- for(int d=0;d<W;d++){- float a=l->ple_conv[(int64_t)d*c->ple_convk+c->ple_convk-1]*norm[d];- for(int k=0;k<c->ple_convk-1;k++)a+=l->ple_conv[(int64_t)d*c->ple_convk+k]*ring[(int64_t)d*state_len+k*c->ngram_size];- out[(int64_t)s*W+d]=gated[d]+q38_silu(a);- float *r=ring+(int64_t)d*state_len;for(int k=0;k<state_len-1;k++)r[k]=r[k+1];r[state_len-1]=norm[d];- }- if(ids[s]==c->eos_id)m->ple_history_len=0;- else if(m->ple_history_len==0){m->ple_history[0]=ids[s];m->ple_history_len=1;}- else if(m->ple_history_len==1){m->ple_history[1]=ids[s];m->ple_history_len=2;}- else {m->ple_history[0]=m->ple_history[1];m->ple_history[1]=ids[s];}- }- free(emb);free(keys);free(value);free(kn);free(qn);free(gated);free(norm);- q38_tm_add(m,Q38_TM_PLE,phase_started);-}--/* Choose a context-independent prefill chunk whose private workspace fits the- * common target. Callers provide exact fixed and per-row byte counts; even a- * hostile-but-valid geometry gets one row rather than an unbounded allocation. */-static int q38_bounded_prefill_rows(int requested,uint64_t fixed,- uint64_t per_row) {- int rows=requested<Q38_PREFILL_BATCH_ROWS?requested:Q38_PREFILL_BATCH_ROWS;- if(rows<1)return 1;- for(;rows>1;rows--)- if(per_row<=UINT64_MAX/(uint64_t)rows&&- fixed<=UINT64_MAX-per_row*(uint64_t)rows&&- fixed+per_row*(uint64_t)rows<=Q38_PREFILL_WORKSPACE_BYTES)- return rows;- return 1;-}--/* Batch the four resident DeltaNet input projections and the output projection- * in bounded chunks. The convolution and recurrent update remain strictly- * token-causal inside each chunk, so chunk boundaries cannot change state or- * floating-point order. */-static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,- float *out) {- double phase_started=now_s();- Cfg *c=&m->c;- int H=c->hidden,VH=c->dn_vheads,KH=c->dn_kheads;- int KD=c->dn_kdim,VD=c->dn_vdim,CD=c->dn_conv_dim,CK=c->dn_convk;- int V=VH*VD,K=KH*KD,rep=VH/KH;- uint64_t row_floats=(uint64_t)CD+2u*(uint64_t)V+2u*(uint64_t)VH;- uint64_t fixed_floats=(uint64_t)CD+2u*(uint64_t)VH*(uint64_t)KD+- (uint64_t)V;- uint64_t row_bytes=row_floats>UINT64_MAX/sizeof(float)?- UINT64_MAX:row_floats*sizeof(float);- uint64_t fixed_bytes=fixed_floats>UINT64_MAX/sizeof(float)?- UINT64_MAX:fixed_floats*sizeof(float);- int rows_capacity=m->prefill_batch?- q38_bounded_prefill_rows(S,fixed_bytes,row_bytes):1;-- float *qkv=falloc((int64_t)rows_capacity*CD);- float *z=falloc((int64_t)rows_capacity*V);- float *bb=falloc((int64_t)rows_capacity*VH);- float *aa=falloc((int64_t)rows_capacity*VH);- float *norm=falloc((int64_t)rows_capacity*V);- float *conv=falloc(CD);- float *q=falloc((int64_t)VH*KD),*k=falloc((int64_t)VH*KD);- float *core=falloc(V);- float *rec=m->DN_rec[layer],*ring=m->DN_conv[layer];-- for(int base=0;base<S;) {- int rows=S-base<rows_capacity?S-base:rows_capacity;- const float *chunk=x+(int64_t)base*H;- q38_dense_matmul(m,qkv,chunk,&l->dn_qkv,rows,H,CD);- q38_dense_matmul(m,z,chunk,&l->dn_z,rows,H,V);- q38_dense_matmul(m,bb,chunk,&l->dn_b,rows,H,VH);- q38_dense_matmul(m,aa,chunk,&l->dn_a,rows,H,VH);-- for(int s=0;s<rows;s++) {- float *qkv_row=qkv+(int64_t)s*CD;- float *z_row=z+(int64_t)s*V;- float *b_row=bb+(int64_t)s*VH;- float *a_row=aa+(int64_t)s*VH;- for(int d=0;d<CD;d++) {- float value=l->dn_conv[(int64_t)d*CK+CK-1]*qkv_row[d];- float *history=ring+(int64_t)d*(CK-1);- for(int tap=0;tap<CK-1;tap++)- value+=l->dn_conv[(int64_t)d*CK+tap]*history[tap];- conv[d]=q38_silu(value);- for(int tap=0;tap<CK-2;tap++)history[tap]=history[tap+1];- history[CK-2]=qkv_row[d];- }- const float *qi=conv,*ki=conv+K,*vi=conv+2*K;- for(int h=0;h<VH;h++) {- float *qh=q+(int64_t)h*KD,*kh=k+(int64_t)h*KD;- memcpy(qh,qi+(int64_t)(h/rep)*KD,(size_t)KD*sizeof(float));- memcpy(kh,ki+(int64_t)(h/rep)*KD,(size_t)KD*sizeof(float));- double qsum=1e-6,ksum=1e-6;- for(int d=0;d<KD;d++) {- qsum+=(double)qh[d]*qh[d];- ksum+=(double)kh[d]*kh[d];- }- float qscale=1.f/sqrtf((float)qsum)/sqrtf((float)KD);- float kscale=1.f/sqrtf((float)ksum);- for(int d=0;d<KD;d++){qh[d]*=qscale;kh[d]*=kscale;}- }- #pragma omp parallel for schedule(static)- for(int h=0;h<VH;h++) {- float *state=rec+(int64_t)h*KD*VD;- const float *qh=q+(int64_t)h*KD;- const float *kh=k+(int64_t)h*KD;- const float *vh=vi+(int64_t)h*VD;- float alpha=expf(-expf(l->dn_alog[h])*- q38_softplus(a_row[h]+l->dn_dtbias[h]));- float beta=q38_sigmoid(b_row[h]);- float delta[512];- int64_t state_cells=(int64_t)KD*VD;- for(int64_t cell=0;cell<state_cells;cell++)state[cell]*=alpha;- for(int value=0;value<VD;value++) {- float previous=0.f;- for(int d=0;d<KD;d++)- previous+=kh[d]*state[(int64_t)d*VD+value];- delta[value]=(vh[value]-previous)*beta;- }- for(int d=0;d<KD;d++)for(int value=0;value<VD;value++)- state[(int64_t)d*VD+value]+=kh[d]*delta[value];- for(int value=0;value<VD;value++) {- float current=0.f;- for(int d=0;d<KD;d++)- current+=qh[d]*state[(int64_t)d*VD+value];- core[(int64_t)h*VD+value]=current;- }- }- float *norm_row=norm+(int64_t)s*V;- for(int h=0;h<VH;h++)- q38_rmsg(norm_row+(int64_t)h*VD,core+(int64_t)h*VD,- z_row+(int64_t)h*VD,l->dn_norm,VD,c->eps,1);- }- q38_dense_matmul(m,out+(int64_t)base*H,norm,&l->dn_out,- rows,V,H);- base+=rows;- }- free(qkv);free(z);free(bb);free(aa);free(norm);free(conv);- free(q);free(k);free(core);- q38_tm_add(m,Q38_TM_DELTANET,phase_started);-}--typedef struct { float score; int block; } Q38Block;-static int q38_block_desc(const void *aa,const void *bb){- const Q38Block *a=(const Q38Block*)aa,*b=(const Q38Block*)bb;- if(a->score>b->score)return -1;if(a->score<b->score)return 1;return a->block-b->block;-}--static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int pos_base,float *out) {- Cfg *c=&m->c;int H=c->hidden,QH=c->q_heads,KVH=c->kv_heads,D=c->head_dim;- int IQ=c->idx_qheads,ID=c->idx_dim,R=c->idx_ratio,maxsel=c->idx_budget+R-1;- float *qp=falloc((int64_t)S*QH*2*D),*kp=falloc((int64_t)S*KVH*D),*vp=falloc((int64_t)S*KVH*D);- float *ip=falloc((int64_t)S*(IQ+c->idx_kheads)*ID);- q38_dense_matmul(m,qp,x,&l->q,S,H,QH*2*D);q38_dense_matmul(m,kp,x,&l->k,S,H,KVH*D);q38_dense_matmul(m,vp,x,&l->v,S,H,KVH*D);- q38_dense_matmul(m,ip,x,&l->idx_qk,S,H,(IQ+c->idx_kheads)*ID);- for(int s=0;s<S;s++){- int pos=pos_base+s;- for(int h=0;h<KVH;h++){- float *kh=kp+(int64_t)s*KVH*D+(int64_t)h*D;q38_rms0(kh,kh,l->kn,D,c->eps);q38_rope(kh,D,c->rotary_dim,pos,c->theta);- memcpy(m->K[layer]+((int64_t)h*m->kv_cap+pos)*D,kh,(size_t)D*sizeof(float));- memcpy(m->V[layer]+((int64_t)h*m->kv_cap+pos)*D,vp+(int64_t)s*KVH*D+(int64_t)h*D,(size_t)D*sizeof(float));- }- memcpy(m->IK[layer]+(int64_t)pos*ID,ip+(int64_t)s*(IQ+1)*ID+(int64_t)IQ*ID,(size_t)ID*sizeof(float));- }- float *heads=falloc((int64_t)S*QH*D),*qidx=falloc((int64_t)IQ*ID),*pool=falloc(ID);- int *selected=(int*)malloc((size_t)maxsel*sizeof(int));- if(!selected){fprintf(stderr,"OOM QSA selection\n");exit(1);}- for(int s=0;s<S;s++){- int pos=pos_base+s,visible=pos+1,blocks=visible/R,tail=blocks*R;- double phase_started=now_s();- for(int h=0;h<IQ;h++){float *qh=qidx+(int64_t)h*ID;memcpy(qh,ip+(int64_t)s*(IQ+1)*ID+(int64_t)h*ID,(size_t)ID*sizeof(float));q38_rms0(qh,qh,l->idx_qn,ID,c->eps);q38_rope(qh,ID,c->rotary_dim,pos,c->theta);}- int take=blocks<c->idx_budget/R?blocks:c->idx_budget/R,nsel=0;- Q38Block *rank=blocks?(Q38Block*)malloc((size_t)blocks*sizeof(Q38Block)):NULL;- if(blocks&&!rank){fprintf(stderr,"OOM QSA block ranking\n");exit(1);}- for(int b=0;b<blocks;b++){- memset(pool,0,(size_t)ID*sizeof(float));for(int r=0;r<R;r++){const float *raw=m->IK[layer]+(int64_t)(b*R+r)*ID;for(int d=0;d<ID;d++)pool[d]+=raw[d]/R;}- q38_rms0(pool,pool,l->idx_kn,ID,c->eps);q38_rope(pool,ID,c->rotary_dim,b*R,c->theta);- float score=0.f;for(int h=0;h<IQ;h++){float a=0.f;for(int d=0;d<ID;d++)a+=qidx[(int64_t)h*ID+d]*pool[d];if(a>0.f)score+=a;}rank[b]=(Q38Block){score/sqrtf((float)ID),b};- }- if(blocks)qsort(rank,(size_t)blocks,sizeof(Q38Block),q38_block_desc);- for(int z=0;z<take;z++)for(int r=0;r<R;r++)selected[nsel++]=rank[z].block*R+r;- for(int t=tail;t<visible;t++)selected[nsel++]=t;free(rank);- q38_tm_add(m,Q38_TM_QSA_INDEX,phase_started); phase_started=now_s();- for(int h=0;h<QH;h++){- float *qraw=qp+(int64_t)s*QH*2*D+(int64_t)h*2*D;- float *qh=falloc(D);memcpy(qh,qraw,(size_t)D*sizeof(float));q38_rms0(qh,qh,l->qn,D,c->eps);q38_rope(qh,D,c->rotary_dim,pos,c->theta);- float *score=falloc(nsel);float mx=-INFINITY;- int khidx=h/(QH/KVH);for(int j=0;j<nsel;j++){const float *kh=m->K[layer]+((int64_t)khidx*m->kv_cap+selected[j])*D;float a=0.f;for(int d=0;d<D;d++)a+=qh[d]*kh[d];a/=sqrtf((float)D);score[j]=a;if(a>mx)mx=a;}- float den=0.f;for(int j=0;j<nsel;j++){score[j]=expf(score[j]-mx);den+=score[j];}- float *oh=heads+(int64_t)s*QH*D+(int64_t)h*D;memset(oh,0,(size_t)D*sizeof(float));- for(int j=0;j<nsel;j++){float a=score[j]/den;const float *vh=m->V[layer]+((int64_t)khidx*m->kv_cap+selected[j])*D;for(int d=0;d<D;d++)oh[d]+=a*vh[d];}- for(int d=0;d<D;d++)oh[d]*=q38_sigmoid(qraw[D+d]);free(qh);free(score);- }- q38_tm_add(m,Q38_TM_QSA_ATTENTION,phase_started);- }- q38_dense_matmul(m,out,heads,&l->o,S,QH*D,H);- free(qp);free(kp);free(vp);free(ip);free(heads);free(qidx);free(pool);free(selected);-}--/* The single-row path is intentionally kept separate from prefill. Decode is- * the latency-sensitive steady state and should not pay for route tables or a- * prompt-sized workspace. */-/* ---- CUDA VRAM expert tier (qwen36_tier.c, fp8 streaming mode) ------------- * The tier keeps its own copies of hot experts in VRAM; the RAM LRU below is- * untouched by it. qt_issue() takes the resident experts of a token's route- * off the CPU, qt_take() adds their outputs back, and every expert the CPU- * did compute is reported with qt_note() so the tier can promote it. The- * tier copies what it needs during qt_note; the slot may be recycled by the- * next token. Only native FP8 slots are reported: gate.data at the slab start- * is what q38_bind_fp8_slot() produces, an expanded slot points elsewhere. */-static void q38_tier_note(int layer,int eid,const Slot *ex) {- if(!qt_ready()||!ex->fp8_slab||ex->gate.data!=ex->fp8_slab)return;- qt_note(layer,eid,(const uint8_t*)ex->gate.data,(const uint8_t*)ex->up.data,- (const uint8_t*)ex->down.data,ex->gate.scales,ex->up.scales,ex->down.scales);-}--/* ---- dense trunk on the GPU (R7b, stage 1) ---------------------------------- * The BF16 trunk is the largest fixed cost of a decode token here (about a- * third), and it is bandwidth-bound. Every matrix of at least 1 MiB is- * offered to the tier's placer by name and layer before qt_init; whatever- * the placer accepts is quantized to int8 per row at start (scale = max|w| /- * 127, the qwen36 dnproj/lmhead format) and uploaded once; the BF16 copy- * stays for prefill and as fallback. Q38_TRUNK_GPU=0 keeps the trunk on the- * CPU (parity runs against the BF16 reference). */-typedef struct { Q38Weight *w; char name[16]; int layer; } Q38TrunkItem;-static Q38TrunkItem *g_trunk; static int g_trunk_n, g_trunk_cap;-static void q38_trunk_add(Q38Weight *w,const char *name,int layer) {- if(!w||!w->data||(w->kind!=Q38_WEIGHT_BF16&&w->kind!=Q38_WEIGHT_F32))return;- size_t bytes=(size_t)w->rows*w->cols+(size_t)w->rows*sizeof(float);- /* Q38_TRUNK_MIN_KB (default 1024): a round trip costs more than a tiny- * GEMV saves; Q38_TRUNK_SKIP=name,name: leave those components on the- * CPU (bisecting a numeric difference, or a component that does not pay) */- static long min_kb=-1; static const char *skip;- if(min_kb<0){ const char *e=getenv("Q38_TRUNK_MIN_KB"); min_kb=e?atol(e):1024; skip=getenv("Q38_TRUNK_SKIP"); }- if(bytes<(size_t)min_kb*1024)return;- if(skip&&*skip){- size_t n=strlen(name); const char *s=skip;- while(*s){ const char *c=strchr(s,','); size_t l=c?(size_t)(c-s):strlen(s);- if(l==n&&!strncmp(s,name,n))return; if(!c)break; s=c+1; }- }- if(g_trunk_n==g_trunk_cap){- g_trunk_cap=g_trunk_cap?2*g_trunk_cap:256;- g_trunk=(Q38TrunkItem*)realloc(g_trunk,(size_t)g_trunk_cap*sizeof(*g_trunk));- if(!g_trunk){fprintf(stderr,"OOM trunk table\n");exit(1);}- }- Q38TrunkItem *it=&g_trunk[g_trunk_n++]; it->w=w; it->layer=layer;- snprintf(it->name,sizeof it->name,"%s",name);- qt_trunk_offer(it->name,layer,bytes);-}-static int q38_trunk_enabled(void) {- const char *e=getenv("Q38_TRUNK_GPU"); return !(e&&e[0]=='0'&&!e[1]);-}-/* offers, before qt_init: lm_head first (the placer takes it first), then the- * layers in order so a partial placement is a prefix of the layers */-static void q38_trunk_offer_all(Model *m) {- if(!q38_trunk_enabled())return;- Cfg *c=&m->c;- q38_trunk_add(&m->lm_head,"lmhead",0);- for(int l=0;l<c->layers;l++){- Layer *L=&m->L[l];- if(c->is_attn[l]){- q38_trunk_add(&L->q,"attnq",l); q38_trunk_add(&L->k,"attnk",l);- q38_trunk_add(&L->v,"attnv",l); q38_trunk_add(&L->o,"attno",l);- q38_trunk_add(&L->idx_qk,"qsaidx",l);- } else {- q38_trunk_add(&L->dn_qkv,"dnqkv",l); q38_trunk_add(&L->dn_z,"dnz",l);- q38_trunk_add(&L->dn_out,"dnout",l);- }- q38_trunk_add(&L->attn_gr.down,"hcad",l); q38_trunk_add(&L->attn_gr.up,"hcau",l);- q38_trunk_add(&L->attn_gr.inject,"hcai",l);- q38_trunk_add(&L->mlp_gr.down,"hcmd",l); q38_trunk_add(&L->mlp_gr.up,"hcmu",l);- q38_trunk_add(&L->mlp_gr.inject,"hcmi",l);- q38_trunk_add(&L->sh_g,"shg",l); q38_trunk_add(&L->sh_u,"shu",l); q38_trunk_add(&L->sh_d,"shd",l);- q38_trunk_add(&L->router,"router",l);- }-}-/* int8 per row, scale = max|w| / 127 (the qwen36 dnproj/lmhead format) */-static void q38_trunk_quantize(const Q38Weight *w,int8_t **qp,float **scp) {- int O=w->rows,I=w->cols;- int8_t *q=(int8_t*)malloc((size_t)O*I); float *sc=(float*)malloc((size_t)O*sizeof(float));- if(!q||!sc){fprintf(stderr,"OOM trunk quantization\n");exit(1);}- #pragma omp parallel for schedule(static)- for(int r=0;r<O;r++){- float row[8192]; float *src=row; float *heap=NULL;- if(I>8192){heap=(float*)malloc((size_t)I*sizeof(float)); src=heap;}- q38_weight_row(w,r,src);- float mx=0.f; for(int k=0;k<I;k++){float a=fabsf(src[k]); if(a>mx)mx=a;}- float s=mx>0.f?mx/127.f:1.f, inv=1.f/s; sc[r]=s;- int8_t *dst=q+(size_t)r*I;- for(int k=0;k<I;k++){int v=(int)lrintf(src[k]*inv); if(v>127)v=127; if(v<-127)v=-127; dst[k]=(int8_t)v;}- free(heap);- }- *qp=q; *scp=sc;-}-/* Q38_TRUNK_CPU_INT8=1: keep the int8 rows on the CPU instead (or as well),- * so the quantization can be judged without a GPU (PPL, token parity) */-static void q38_trunk_cpu_int8(Model *m) {- const char *e=getenv("Q38_TRUNK_CPU_INT8");- if(!e||e[0]!='1'||e[1])return;- if(!g_trunk_n) q38_trunk_offer_all(m);- double t0=now_s(); size_t bytes=0;- for(int i=0;i<g_trunk_n;i++){- Q38Weight *w=g_trunk[i].w; if(w->q8)continue;- q38_trunk_quantize(w,&w->q8,&w->q8sc); bytes+=(size_t)w->rows*w->cols;- }- fprintf(stderr,"[qwen38] trunk: %d matrices int8 on the CPU (%.2f GiB) in %.1fs (Q38_TRUNK_CPU_INT8)\n",- g_trunk_n,bytes/1073741824.0,now_s()-t0);-}-/* after qt_init: quantize and upload what the placer accepted */-static void q38_trunk_place_all(Model *m) {- (void)m;- int placed=0; size_t placed_bytes=0; double t0=now_s();- for(int i=0;i<g_trunk_n;i++){- Q38TrunkItem *it=&g_trunk[i]; Q38Weight *w=it->w;- int dev=qt_place_of(it->name,it->layer);- if(dev==QT_PLACE_CPU)continue;- int O=w->rows,I=w->cols;- int8_t *q; float *sc; q38_trunk_quantize(w,&q,&sc);- int h=qt_dense_init(q,sc,I,O,dev);- if(h>=0&&getenv("Q38_TRUNK_SELFTEST")){- /* DIAG: GPU int8 GEMV against the same int8 matrix on the CPU */- float *x=(float*)malloc((size_t)I*sizeof(float)),*yg=(float*)malloc((size_t)O*sizeof(float)),*yc=(float*)malloc((size_t)O*sizeof(float));- for(int k=0;k<I;k++)x[k]=sinf(0.37f*k)+0.1f*(k%7);- for(int r=0;r<O;r++){double a=0; for(int k=0;k<I;k++)a+=(double)q[(size_t)r*I+k]*x[k]; yc[r]=(float)(a*sc[r]);}- int ok=qt_dense_matmul(h,yg,x,I,O); double num=0,den=0; int worst=0;- for(int r=0;r<O;r++){double d=yg[r]-yc[r]; num+=d*d; den+=(double)yc[r]*yc[r]; if(fabs(d)>fabs(yg[worst]-yc[worst]))worst=r;}- fprintf(stderr,"[selftest] %-7s L%-2d [O=%d I=%d] ok=%d rel.err %.2e worst row %d gpu %.5g cpu %.5g\n",it->name,it->layer,O,I,ok,den>0?sqrt(num/den):-1.0,worst,yg[worst],yc[worst]);- free(x);free(yg);free(yc);- }- free(q); free(sc);- if(h>=0){ w->gpu=h+1; placed++; placed_bytes+=(size_t)O*I; }- }- if(g_trunk_n)- fprintf(stderr,"[qtier] qwen38 trunk: %d of %d offered matrices resident as int8 (%.2f GiB) in %.1fs; the rest stays BF16 on the CPU\n",- placed,g_trunk_n,placed_bytes/1073741824.0,now_s()-t0);-}--/* Start the tier after the model is loaded. COLI_CUDA=1 turns it on (the- * tier reads that itself); it needs every layer's experts in native FP8 with- * the block-scale bank resident, because the GPU kernels consume exactly the- * checkpoint layout (e4m3 bytes + [ceil(O/128), ceil(I/128)] scales). */-static void q38_tier_start(Model *m,int cap) {- const char *on=getenv("COLI_CUDA");- if(!on||on[0]!='1'||on[1])return;- Cfg *c=&m->c;- if(!m->native_fp8){- fprintf(stderr,"[qtier] qwen38: expert tier needs native FP8 experts (Q38_NATIVE_FP8=1); staying on the CPU\n");- return;- }- for(int layer=0;layer<c->layers;layer++)- if(!q38_prepare_expert_scale_bank(m,layer)){- fprintf(stderr,"[qtier] qwen38: layer %d experts are not native FP8; staying on the CPU\n",layer);- return;- }- q38_trunk_offer_all(m); /* sizes only; the placer decides in qt_init */- if(qt_init_fp8(c->layers,c->experts,c->hidden,c->inter,cap,c->topk,E4M3_LUT)){- atexit(qt_shutdown);- fprintf(stderr,"[qtier] qwen38: fp8 expert tier on (RAM LRU %d/layer stays; experts stream to VRAM as they get hot)\n",cap);- q38_trunk_place_all(m);- }-}--static void q38_moe_decode(Model *m,Layer *l,int layer,const float *x,int S,float *out) {- Cfg *c=&m->c;int H=c->hidden,E=c->experts,K=c->topk,I=c->inter,SI=c->shared_inter;- float *logits=falloc(E),*sg=falloc(SI),*su=falloc(SI),*sh=falloc(SI),*shared=falloc(H);- float *eg=falloc(I),*eu=falloc(I),*eh=falloc(I),*eo=falloc(H);- for(int s=0;s<S;s++){- const float *xs=x+(int64_t)s*H;float *ys=out+(int64_t)s*H;memset(ys,0,(size_t)H*sizeof(float));- q38_dense_matmul(m,logits,xs,&l->router,1,H,E);float mx=logits[0];for(int e=1;e<E;e++)if(logits[e]>mx)mx=logits[e];- double all=0;for(int e=0;e<E;e++){logits[e]=expf(logits[e]-mx);all+=logits[e];}- int idx[Q38_MAX_TOPK];float val[Q38_MAX_TOPK];- for(int z=0;z<K;z++){int best=-1;float bv=-1.f;for(int e=0;e<E;e++){int used=0;for(int j=0;j<z;j++)if(idx[j]==e)used=1;if(!used&&logits[e]>bv){bv=logits[e];best=e;}}idx[z]=rt_router_pick(best,z,E,layer);val[z]=logits[idx[z]];}- double top=0;for(int z=0;z<K;z++)top+=val[z];double den=c->norm_topk?top:all;- float route_gates[Q38_MAX_TOPK];- for(int z=0;z<K;z++) route_gates[z]=(float)(val[z]/den);- rt_route(layer,s,idx,route_gates,K); /* shared counts + post-normalization trace */- /* Resident experts run on the GPU from here on (asynchronously); the- * CPU only loads and computes the rest, in route order. */- uint32_t qmask=qt_issue(layer,idx,K,xs);- int cpu_idx[Q38_MAX_TOPK],cpu_rank[Q38_MAX_TOPK],cpu_n=0;- for(int z=0;z<K;z++)if(!((qmask>>z)&1u)){cpu_idx[cpu_n]=idx[z];cpu_rank[cpu_n]=z;cpu_n++;}- q38_prefetch_native_fp8_experts(m,layer,cpu_idx,cpu_n);- double phase_started=now_s();- q38_weight_matmul(sg,xs,&l->sh_g,1,H,SI);q38_weight_matmul(su,xs,&l->sh_u,1,H,SI);- for(int j=0;j<SI;j++)sh[j]=q38_silu(sg[j])*su[j];q38_weight_matmul(shared,sh,&l->sh_d,1,SI,H);- float gate=0.f;for(int d=0;d<H;d++)gate+=xs[d]*l->sh_gate[d];gate=q38_sigmoid(gate);- q38_tm_add(m,Q38_TM_SHARED_EXPERT,phase_started);- Slot *selected[Q38_MAX_TOPK];- int loaded_batch=q38_expert_get_batch(m,layer,cpu_idx,cpu_n,selected);- for(int i=0;i<cpu_n;i++){- int z=cpu_rank[i];- Slot *ex=loaded_batch?selected[i]:q38_expert_get(m,layer,cpu_idx[i]);phase_started=now_s();- q38_weight_matmul(eg,xs,&ex->gate,1,H,I);q38_weight_matmul(eu,xs,&ex->up,1,H,I);- for(int j=0;j<I;j++)eh[j]=q38_silu(eg[j])*eu[j];q38_weight_matmul(eo,eh,&ex->down,1,I,H);- for(int d=0;d<H;d++)ys[d]+=route_gates[z]*eo[d];- q38_tm_add(m,Q38_TM_ROUTED_EXPERT,phase_started);- q38_tier_note(layer,cpu_idx[i],ex);- }- /* GPU experts land after the CPU ones: same values, one more group in- * the float sum (that is the only ordering difference to a CPU run). */- qt_take(qmask,route_gates,K,ys);- for(int d=0;d<H;d++)ys[d]+=gate*shared[d];- }- rt_trace_end();- free(logits);free(sg);free(su);free(sh);free(shared);free(eg);free(eu);free(eh);free(eo);-}--typedef struct {- int expert;- float gate;-} Q38RouteAssignment;--/* Pick a prefill size from a fixed workspace budget. The number of rows is- * deliberately bounded independently of the context length: a long prompt- * therefore reuses the same route, expert and matmul buffers one chunk at a- * time. A single assignment needs input, gate/up activations and output;- * these are the only buffers that scale with the number of routed rows. */-static int q38_moe_prefill_rows(const Cfg *c,int requested) {- int64_t H=c->hidden,I=c->inter,E=c->experts,K=c->topk,SI=c->shared_inter;- uint64_t per_assignment=(uint64_t)(2LL*((int64_t)H+I))*sizeof(float)+- sizeof(Q38RouteAssignment)+2*sizeof(int);- uint64_t per_row=(uint64_t)E*sizeof(float)+- (uint64_t)(3LL*(int64_t)SI+H+1)*sizeof(float);- uint64_t fixed=(uint64_t)E*(4*sizeof(int)+2*sizeof(Slot*));- if(fixed<=UINT64_MAX-sizeof(int))fixed+=sizeof(int); /* group_offsets[E] */- else fixed=UINT64_MAX;- uint64_t assignment_row=(uint64_t)K>UINT64_MAX/per_assignment?- UINT64_MAX:(uint64_t)K*per_assignment;- uint64_t total_row=per_row>UINT64_MAX-assignment_row?- UINT64_MAX:per_row+assignment_row;- return q38_bounded_prefill_rows(requested,fixed,total_row);-}--/* Prefill MoE: route a bounded row chunk first, then execute each distinct- * expert's assignments as one batched SwiGLU. Grouping is an I/O optimization- * only. Expert outputs are placed back in assignment order and the final- * weighted reduction still visits rank 0..top-k-1 for every row, preserving the- * decode path's floating-point accumulation order. */-static void q38_moe_prefill(Model *m,Layer *l,int layer,const float *x,- int S,float *out) {- Cfg *c=&m->c;- int H=c->hidden,E=c->experts,K=c->topk,I=c->inter,SI=c->shared_inter;- int rows_capacity=q38_moe_prefill_rows(c,S);- int64_t max_assign=(int64_t)rows_capacity*K;-- Q38RouteAssignment *routes=(Q38RouteAssignment*)malloc(- (size_t)max_assign*sizeof(*routes));- int *assignments=(int*)malloc((size_t)max_assign*sizeof(*assignments));- int *assignment_positions=(int*)malloc((size_t)max_assign*sizeof(*assignment_positions));- int *group_counts=(int*)calloc((size_t)E,sizeof(*group_counts));- int *group_offsets=(int*)malloc((size_t)(E+1)*sizeof(*group_offsets));- int *group_cursor=(int*)malloc((size_t)E*sizeof(*group_cursor));- int *unique=(int*)malloc((size_t)E*sizeof(*unique));- Slot **batch_slots=(Slot**)calloc((size_t)E,sizeof(*batch_slots));- if(!routes||!assignments||!assignment_positions||!group_counts||- !group_offsets||!group_cursor||!unique||!batch_slots){- fprintf(stderr,"OOM Qwen3.8 MoE prefill metadata\n");exit(1);- }-- float *logits=falloc((int64_t)rows_capacity*E);- float *shared_gate=falloc(rows_capacity);- float *shared_g=falloc((int64_t)rows_capacity*SI);- float *shared_u=falloc((int64_t)rows_capacity*SI);- float *shared_hidden=falloc((int64_t)rows_capacity*SI);- float *shared_out=falloc((int64_t)rows_capacity*H);- float *expert_input=falloc(max_assign*H);- float *expert_gate=falloc(max_assign*I);- float *expert_up=falloc(max_assign*I);- float *routed_out=falloc(max_assign*H);-- for(int base=0;base<S;) {- int rows=S-base<rows_capacity?S-base:rows_capacity;- int assignment_count=rows*K;- memset(group_counts,0,(size_t)E*sizeof(*group_counts));-- /* Route the complete chunk with one resident router matmul. Selection- * intentionally mirrors q38_moe_decode, including rt_router_pick's- * deterministic fallback for invalid logits. */- q38_dense_matmul(m,logits,x+(int64_t)base*H,&l->router,rows,H,E);- for(int s=0;s<rows;s++) {- float *probabilities=logits+(int64_t)s*E;- float maximum=probabilities[0];- for(int e=1;e<E;e++)if(probabilities[e]>maximum)maximum=probabilities[e];- double total=0.0;- for(int e=0;e<E;e++) {- probabilities[e]=expf(probabilities[e]-maximum);- total+=probabilities[e];- }- int selected[Q38_MAX_TOPK];- float selected_probability[Q38_MAX_TOPK];- for(int rank=0;rank<K;rank++) {- int best=-1;float best_value=-1.f;- for(int e=0;e<E;e++) {- int used=0;- for(int previous=0;previous<rank;previous++)- if(selected[previous]==e){used=1;break;}- if(!used&&probabilities[e]>best_value){- best_value=probabilities[e];best=e;- }- }- selected[rank]=rt_router_pick(best,rank,E,layer);- selected_probability[rank]=probabilities[selected[rank]];- }- double top=0.0;- for(int rank=0;rank<K;rank++)top+=selected_probability[rank];- double denominator=c->norm_topk?top:total;- float gates[Q38_MAX_TOPK];- for(int rank=0;rank<K;rank++) {- gates[rank]=(float)(selected_probability[rank]/denominator);- int assignment=s*K+rank;- routes[assignment]=(Q38RouteAssignment){selected[rank],gates[rank]};- group_counts[selected[rank]]++;- }- rt_route(layer,base+s,selected,gates,K);- }-- /* Prefixing by expert makes every group contiguous while the inverse- * map lets the final reduction recover the original row/rank order. */- group_offsets[0]=0;- int unique_count=0;- for(int e=0;e<E;e++) {- group_offsets[e+1]=group_offsets[e]+group_counts[e];- if(group_counts[e])unique[unique_count++]=e;- }- memset(group_cursor,0,(size_t)E*sizeof(*group_cursor));- for(int assignment=0;assignment<assignment_count;assignment++) {- int expert=routes[assignment].expert;- int position=group_offsets[expert]+group_cursor[expert]++;- assignments[position]=assignment;- assignment_positions[assignment]=position;- }-- /* One advice range per distinct expert is enough for this chunk. */- q38_prefetch_native_fp8_experts(m,layer,unique,unique_count);-- /* Shared expert work is independent across rows and remains resident;- * batching it here also keeps its cost out of the routed groups. */- double phase_started=now_s();- q38_weight_matmul(shared_g,x+(int64_t)base*H,&l->sh_g,rows,H,SI);- q38_weight_matmul(shared_u,x+(int64_t)base*H,&l->sh_u,rows,H,SI);- for(int s=0;s<rows;s++)- for(int j=0;j<SI;j++)- shared_hidden[(int64_t)s*SI+j]=- q38_silu(shared_g[(int64_t)s*SI+j])*- shared_u[(int64_t)s*SI+j];- q38_weight_matmul(shared_out,shared_hidden,&l->sh_d,rows,SI,H);- for(int s=0;s<rows;s++) {- const float *xs=x+(int64_t)(base+s)*H;- float gate=0.f;- for(int d=0;d<H;d++)gate+=xs[d]*l->sh_gate[d];- shared_gate[s]=q38_sigmoid(gate);- }- q38_tm_add(m,Q38_TM_SHARED_EXPERT,phase_started);-- /* A demand-set batch is particularly effective for native FP8: reserve- * all slots before workers read the two coalesced ranges. Other native- * layouts use the same grouped matmul loop with the ordinary bounded- * LRU loader, so no format loses correctness or caching. */- memset(out+(int64_t)base*H,0,(size_t)rows*H*sizeof(float));- /* A prompt chunk can route to more distinct experts than the retained- * cache can hold. Load and consume cache-sized groups instead of- * declining the complete parallel demand set: every expert is still- * loaded once for this chunk, and a later group may safely reuse its- * slots because the preceding outputs already live in routed_out. */- int load_limit=m->cache[layer].cap;- if(load_limit>Q38_MAX_TOPK)load_limit=Q38_MAX_TOPK;- if(load_limit<1)load_limit=1;- for(int unique_base=0;unique_base<unique_count;) {- int load_count=unique_count-unique_base;- if(load_count>load_limit)load_count=load_limit;- int loaded_batch=load_count>=2&&q38_expert_get_batch(- m,layer,unique+unique_base,load_count,batch_slots);- for(int offset=0;offset<load_count;offset++) {- int e=unique[unique_base+offset];- int count=group_counts[e];- Slot *expert=loaded_batch?batch_slots[offset]:- q38_expert_get(m,layer,e);- int first=group_offsets[e];- for(int a=0;a<count;a++) {- int assignment=assignments[first+a];- int row=assignment/K;- memcpy(expert_input+(int64_t)a*H,- x+(int64_t)(base+row)*H,(size_t)H*sizeof(float));- }- phase_started=now_s();- q38_weight_matmul(expert_gate,expert_input,&expert->gate,- count,H,I);- q38_weight_matmul(expert_up,expert_input,&expert->up,count,H,I);- for(int a=0;a<count;a++)for(int j=0;j<I;j++)- expert_gate[(int64_t)a*I+j]=- q38_silu(expert_gate[(int64_t)a*I+j])*- expert_up[(int64_t)a*I+j];- q38_weight_matmul(routed_out+(int64_t)first*H,expert_gate,- &expert->down,count,I,H);- q38_tm_add(m,Q38_TM_ROUTED_EXPERT,phase_started);- }- unique_base+=load_count;- }-- /* The grouped execution above is intentionally not the reduction- * order. Replaying the original top-k sequence gives the same result- * as the single-row implementation for F32, BF16 and block-FP8. */- for(int s=0;s<rows;s++) {- float *ys=out+(int64_t)(base+s)*H;- for(int rank=0;rank<K;rank++) {- int assignment=s*K+rank;- const float *expert_output=routed_out+- (int64_t)assignment_positions[assignment]*H;- float gate=routes[assignment].gate;- for(int d=0;d<H;d++)ys[d]+=gate*expert_output[d];- }- const float *shared=shared_out+(int64_t)s*H;- for(int d=0;d<H;d++)ys[d]+=shared_gate[s]*shared[d];- }- base+=rows;- }- rt_trace_end();- free(logits);free(shared_gate);free(shared_g);free(shared_u);- free(shared_hidden);free(shared_out);free(expert_input);free(expert_gate);- free(expert_up);free(routed_out);free(routes);free(assignments);- free(assignment_positions);free(group_counts);free(group_offsets);- free(group_cursor);free(unique);free(batch_slots);-}--static void q38_moe(Model *m,Layer *l,int layer,const float *x,int S,float *out) {- if(S<=1||!m->prefill_batch)q38_moe_decode(m,l,layer,x,S,out);- else q38_moe_prefill(m,l,layer,x,S,out);-}--static void reset_recurrent(Model *m) {- Cfg *c=&m->c;- for(int i=0;i<c->layers;i++)if(!c->is_attn[i]){- memset(m->DN_rec[i],0,(size_t)c->dn_vheads*c->dn_kdim*c->dn_vdim*sizeof(float));- memset(m->DN_conv[i],0,(size_t)c->dn_conv_dim*(c->dn_convk-1)*sizeof(float));- }- memset(m->PLE_conv_state,0,(size_t)c->hc_width*(c->ple_convk-1)*c->ngram_size*sizeof(float));m->ple_history_len=0;-}--static void ensure_kv(Model *m) {- Cfg *c=&m->c;if(m->max_t<=m->kv_cap&&m->K)return;- if(m->K){for(int i=0;i<c->layers;i++){free(m->K[i]);free(m->V[i]);free(m->IK[i]);}free(m->K);free(m->V);free(m->IK);}- m->K=(float**)calloc((size_t)c->layers,sizeof(float*));m->V=(float**)calloc((size_t)c->layers,sizeof(float*));m->IK=(float**)calloc((size_t)c->layers,sizeof(float*));- for(int i=0;i<c->layers;i++)if(c->is_attn[i]){- m->K[i]=falloc((int64_t)c->kv_heads*m->max_t*c->head_dim);m->V[i]=falloc((int64_t)c->kv_heads*m->max_t*c->head_dim);m->IK[i]=falloc((int64_t)m->max_t*c->idx_dim);- }- m->kv_cap=m->max_t;-}--/* Run only the requested native layer interval over hyper-residual activations.- * Segment callers supply the four-stream boundary state directly; unlike step,- * this path deliberately does not gather embeddings or apply the final mixer. */-static void q38_layers_forward_range(Model *m,float *hyper,const int *ids,- int S,int pos_base,int layer_begin,- int layer_end) {- Cfg *c=&m->c; int H=c->hidden,W=c->hc_width,C=c->hc_count;- float *mixed=falloc((int64_t)S*H),*inject=falloc((int64_t)S*C),*block=falloc((int64_t)S*H);- for(int i=layer_begin;i<layer_end;i++){- Layer *l=&m->L[i];- if(i==c->ple_layer){- float *ple=falloc((int64_t)S*W); q38_ple(m,ids,S,hyper,ple);- for(int64_t z=0;z<(int64_t)S*W;z++) hyper[z]+=ple[z];- free(ple);- }- q38_gr_read(m,&l->attn_gr,hyper,S,mixed,inject);- if(c->is_attn[i]) q38_attention(m,l,i,mixed,S,pos_base,block);- else q38_deltanet(m,l,i,mixed,S,block);- q38_gr_apply(c,hyper,block,inject,S);- q38_gr_read(m,&l->mlp_gr,hyper,S,mixed,inject);- q38_moe(m,l,i,mixed,S,block);- q38_gr_apply(c,hyper,block,inject,S);- }- free(mixed); free(inject); free(block);-}--/* Canale logprobs (modalita jev). Qui la fotografia dello stato NON si- * aggiunge: questo motore ce l'ha gia, si chiama Q38PrefixCache e salva stato- * ricorrente, id e logit finali dopo ogni prompt. Manca solo la lettura, cioe- * un passaggio di lm_head per posizione invece che solo sull'ultima, e il- * predittore del primo token fresco, che sono i logit gia salvati. */-static int g_echo_k = 0;-static const char *g_echo_id = NULL;-static const float *g_echo_pin_logit = NULL; /* logit dell'ultima posizione riusata */--static void q38_echo(const char *id, int pos, int token, const float *lo, int V, int k){- char tail[1024]; coli_logprob_tail(tail, sizeof tail, lo, V, token, k);- unsigned char *piece=NULL; int n=0;- if(decode_id_alloc(token,&piece,&n)) { piece=NULL; n=0; }- if(n<0) n=0;- printf("ECHO %s %d %d%s\n", id, n, pos, tail);- if(n>0) fwrite(piece,1,(size_t)n,stdout);- fputc('\n', stdout); fflush(stdout);- free(piece);-}--static float *step(Model *m,const int *ids,int S,int pos_base) {- Cfg *c=&m->c;int H=c->hidden,W=c->hc_width,C=c->hc_count;- m->timers.forwards++;- float *hyper=falloc((int64_t)S*W);- /* Le righe PLE partono ADESSO, non al layer 2 dove servono: sono note dagli- * id dei token soltanto, e i due layer che le precedono danno il tempo di- * farle arrivare dal disco. */- q38_ple_prefetch(m,ids,S);- for(int s=0;s<S;s++){- if(ids[s]<0||ids[s]>=c->vocab){fprintf(stderr,"token id %d outside vocabulary\n",ids[s]);exit(1);}- float *e=hyper+(int64_t)s*W;- int abs_pos=pos_base+s, vis_row=-1;- if(m->vis_map&&abs_pos>=0&&abs_pos<m->vis_map_len) vis_row=m->vis_map[abs_pos];- if(vis_row>=0&&vis_row<m->vis_rows_n)- memcpy(e,m->vis_rows+(int64_t)vis_row*H,(size_t)H*sizeof(float));- else q38_weight_row(&m->embed,ids[s],e);- for(int b=1;b<C;b++)memcpy(e+(int64_t)b*H,e,(size_t)H*sizeof(float));- }- float *mixed=falloc((int64_t)S*H),*inject=falloc((int64_t)S*C),*block=falloc((int64_t)S*H);- for(int i=0;i<c->layers;i++){- Layer *l=&m->L[i];- if(i==c->ple_layer){float *ple=falloc((int64_t)S*W);q38_ple(m,ids,S,hyper,ple);for(int64_t z=0;z<(int64_t)S*W;z++)hyper[z]+=ple[z];free(ple);- /* consumate: il chunk successivo ha altri token, e riusare queste- * righe darebbe gli embedding del chunk precedente combaciando in- * silenzio invece di dare errore. */- free(m->ple_pref); m->ple_pref=NULL; m->ple_pref_rows=0;}- q38_gr_read(m,&l->attn_gr,hyper,S,mixed,inject);- if(c->is_attn[i])q38_attention(m,l,i,mixed,S,pos_base,block);else q38_deltanet(m,l,i,mixed,S,block);- q38_gr_apply(c,hyper,block,inject,S);- q38_gr_read(m,&l->mlp_gr,hyper,S,mixed,inject);q38_moe(m,l,i,mixed,S,block);q38_gr_apply(c,hyper,block,inject,S);- }- q38_gr_read(m,&m->final_gr,hyper,S,mixed,NULL);m->kv_len=pos_base+S;- float *logit=falloc(c->vocab);double phase_started=now_s();- /* Lettura del prefill: la posizione p predice il token p+1. Il primo token- * fresco lo predice la fotografia del prefisso, quando c'e. Pagata solo da- * chi ha chiesto il canale. */- if(g_echo_k>0&&g_echo_id&&S>0){- if(g_echo_pin_logit)- q38_echo(g_echo_id,pos_base,ids[0],g_echo_pin_logit,c->vocab,g_echo_k);- for(int p=0;p+1<S;p++){- q38_weight_matmul(logit,mixed+(int64_t)p*H,&m->lm_head,1,H,c->vocab);- q38_echo(g_echo_id,pos_base+p+1,ids[p+1],logit,c->vocab,g_echo_k);- }- }- q38_weight_matmul(logit,mixed+(int64_t)(S-1)*H,&m->lm_head,1,H,c->vocab);- q38_tm_add(m,Q38_TM_LM_HEAD,phase_started);- free(hyper);free(mixed);free(inject);free(block);return logit;-}--static int q38_tm_enabled(void) {- const char *enabled=getenv("COLI_TIMERS");- return enabled&&enabled[0]=='1'&&enabled[1]=='\0';-}--static void q38_tm_report_bank(const Q38Timers *timers,const char *scope) {- if(!q38_tm_enabled())return;- static const char *names[Q38_TM_COUNT]={- "expert-read","fp8-expand","routed-expert","shared-expert",- "resident-mm","deltanet","qsa-index","qsa-attn","ple","lm-head"- };- double per=timers->forwards?1000.0/timers->forwards:0.0;- fprintf(stderr,"[qwen38 timers] %s: %llu forwards\n",scope,- (unsigned long long)timers->forwards);- for(int i=0;i<Q38_TM_COUNT;i++)- fprintf(stderr,"[qwen38 timers] %-14s %9.3f s %9.3f ms/forward\n",- names[i],timers->seconds[i],timers->seconds[i]*per);- fprintf(stderr,"[qwen38 timers] architecture phases overlap resident-mm; "- "expert-read is disk service and fp8-expand is synchronous miss work\n");-}--/* Snapshot of the timer bank taken when the prompt's forward has finished- * (generate() sets it), so a "decode" bank can be reported apart from the- * prefill. Without it the per-forward figures of a long prompt are prefill- * work divided by decode forwards, which reads like a decode budget and is- * not one. */-static Q38Timers g_tm_prefill_snapshot; static int g_tm_have_snapshot;-static void q38_tm_snapshot_prefill(const Model *m) {- g_tm_prefill_snapshot=m->timers; g_tm_have_snapshot=1;-}-static void tm_report(const Model *m) {- q38_tm_report_bank(&m->timers,"total");- if(g_tm_have_snapshot&&q38_tm_enabled()){- Q38Timers decode=m->timers;- for(int i=0;i<Q38_TM_COUNT;i++)decode.seconds[i]-=g_tm_prefill_snapshot.seconds[i];- decode.forwards-=g_tm_prefill_snapshot.forwards;- q38_tm_report_bank(&g_tm_prefill_snapshot,"prefill");- q38_tm_report_bank(&decode,"decode");- }- if(q38_tm_enabled())- fprintf(stderr,"[qwen38 expert I/O] weight-ranges=%llu scale-ranges=%llu "- "coalesced-gate-up=%llu prefetched=%llu parallel-batches=%llu "- "resident-scales=%.2f MiB\n",- (unsigned long long)m->expert_weight_reads,- (unsigned long long)m->expert_scale_reads,- (unsigned long long)m->expert_pair_reads,- (unsigned long long)m->expert_prefetch_ranges,- (unsigned long long)m->expert_parallel_batches,- m->expert_scale_bytes/1048576.0);-}--static void q38_layer_free(Layer *l) {- if(!l) return;- free(l->attn_gr.norm);q38_weight_free(&l->attn_gr.down);q38_weight_free(&l->attn_gr.up);q38_weight_free(&l->attn_gr.inject);- free(l->mlp_gr.norm);q38_weight_free(&l->mlp_gr.down);q38_weight_free(&l->mlp_gr.up);q38_weight_free(&l->mlp_gr.inject);- q38_weight_free(&l->router);q38_weight_free(&l->sh_g);q38_weight_free(&l->sh_u);q38_weight_free(&l->sh_d);free(l->sh_gate);- q38_weight_free(&l->q);q38_weight_free(&l->k);q38_weight_free(&l->v);q38_weight_free(&l->o);free(l->qn);free(l->kn);- q38_weight_free(&l->idx_qk);free(l->idx_qn);free(l->idx_kn);- q38_weight_free(&l->dn_qkv);q38_weight_free(&l->dn_z);q38_weight_free(&l->dn_b);q38_weight_free(&l->dn_a);free(l->dn_conv);- free(l->dn_dtbias);free(l->dn_alog);free(l->dn_norm);q38_weight_free(&l->dn_out);- q38_weight_free(&l->ple_key);q38_weight_free(&l->ple_value);free(l->ple_norm_key);free(l->ple_norm_query);- free(l->ple_norm_conv); free(l->ple_conv);- memset(l,0,sizeof(*l));-}--static void q38_model_free(Model *m) {- if(!m) return;- for(int i=0;i<m->c.layers;i++) {- if(m->L)q38_layer_free(&m->L[i]);- if(m->cache) {- if(m->cache[i].slots) {- for(int s=0;s<m->cache[i].n;s++) {- q38_weight_free(&m->cache[i].slots[s].gate);- q38_weight_free(&m->cache[i].slots[s].up);- q38_weight_free(&m->cache[i].slots[s].down);- free(m->cache[i].slots[s].fp8_slab);- }- }- free(m->cache[i].slots); free(m->cache[i].by_expert);- }- if(m->expert_scales)free(m->expert_scales[i].values);- free(m->DN_rec ? m->DN_rec[i] : NULL); free(m->DN_conv ? m->DN_conv[i] : NULL);- free(m->K ? m->K[i] : NULL); free(m->V ? m->V[i] : NULL); free(m->IK ? m->IK[i] : NULL);- }- free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);- q38_weight_free(&m->embed);q38_weight_free(&m->lm_head);- free(m->final_gr.norm);q38_weight_free(&m->final_gr.down);q38_weight_free(&m->final_gr.up);q38_weight_free(&m->final_gr.inject);- free(m->ple_history); free(m->PLE_conv_state); free(m->c.is_attn); st_destroy(&m->S);- memset(m,0,sizeof(*m));-}--#endif /* COLI_QWEN38_CORE_H */+ }+ unsigned char *protected_slots=(unsigned char*)calloc((size_t)cache->cap,1);+ if(!protected_slots){fprintf(stderr,"OOM expert batch reservations\n");exit(1);}+ for(int index=0;index<count;index++){+ int slot_index=cache->by_expert[experts[index]];+ if(slot_index>=0)protected_slots[slot_index]=1;+ }+ int job_count=0;+ for(int index=0;index<count;index++){+ int expert=experts[index],slot_index=cache->by_expert[expert];Slot *slot;+ if(slot_index>=0){+ slot=&cache->slots[slot_index];m->hits++;+ }else{+ m->miss++;+ if(cache->n<cache->cap){+ slot=&cache->slots[cache->n++];slot->eid=-1;+ }else{+ int victim=-1;+ for(int candidate=0;candidate<cache->n;candidate++)+ if(!protected_slots[candidate]&&+ (victim<0||cache->slots[candidate].used<cache->slots[victim].used))+ victim=candidate;+ if(victim<0){+ fprintf(stderr,"Qwen3.8 expert demand set has no reservable cache slot\n");+ exit(1);+ }+ slot=&cache->slots[victim];+ if(slot->eid>=0)cache->by_expert[slot->eid]=-1;+ }+ slot->eid=-1;jobs[job_count].expert=expert;jobs[job_count].slot=slot;+ if(!q38_native_fp8_expert_tensors(m,layer,expert,jobs[job_count].weight)){+ fprintf(stderr,"Qwen3.8 expert layout changed during batch reservation\n");exit(1);+ }+ job_count++;+ }+ slot->used=++m->clock;selected[index]=slot;+ protected_slots[slot-cache->slots]=1;+ }+ free(protected_slots);+ if(job_count){+ int workers=job_count;+#ifdef _OPENMP+ int thread_limit=omp_get_max_threads();if(workers>thread_limit)workers=thread_limit;+#else+ (void)workers; /* only the pragma below reads it, and that is gone without OpenMP */+#endif+ double started=now_s();+ #pragma omp parallel for schedule(static) num_threads(workers) if(job_count>1)+ for(int job=0;job<job_count;job++)+ q38_load_native_fp8_ranges(m,layer,jobs[job].expert,jobs[job].slot,+ jobs[job].weight);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ m->expert_weight_reads+=(uint64_t)job_count*2;+ m->expert_pair_reads+=(uint64_t)job_count;+ if(job_count>1)m->expert_parallel_batches++;+ for(int job=0;job<job_count;job++){+ Slot *slot=jobs[job].slot;slot->eid=jobs[job].expert;+ cache->by_expert[jobs[job].expert]=(int)(slot-cache->slots);+ }+ }+ return 1;+}++static void q38_ple_row(Model *m,int64_t row,float *out) {+ Cfg *c=&m->c; int p=0;+ while(p+1<m->ple_part_count&&row>=m->ple_part_start[p+1])p++;+ if(p>=m->ple_part_count||row<m->ple_part_start[p]){fprintf(stderr,"PLE row out of range: %lld\n",(long long)row);exit(1);}+ int64_t local=row-m->ple_part_start[p]; st_tensor *t=m->ple_parts[p]; const char *nm=m->ple_part_names[p];+ Q38_NEED(c->ngram_head_dim>0&&local>=0&&local<=INT64_MAX/c->ngram_head_dim,+ "PLE row byte offset overflows");+ if(t->dtype==4){+ uint8_t raw[512]; Q38_NEED(c->ngram_head_dim<=(int)sizeof raw,"PLE row too wide");+ st_read_slice_raw_cap(&m->S,nm,local*c->ngram_head_dim,c->ngram_head_dim,raw,sizeof raw,1);+ for(int d=0;d<c->ngram_head_dim;d++)out[d]=e4m3_decode(raw[d])*m->ple_weight_scale;+ } else st_read_slice_f32(&m->S,nm,local*c->ngram_head_dim,c->ngram_head_dim,out,1);+}++static int64_t q38_hash_row(Model *m,int head,int ngram,int64_t cur,int64_t p1,int64_t p2) {+ uint64_t x=(uint64_t)cur*(uint64_t)m->ple_multipliers[0];+ x^=(uint64_t)p1*(uint64_t)m->ple_multipliers[1];+ if(ngram==3)x^=(uint64_t)p2*(uint64_t)m->ple_multipliers[2];+ int64_t sx=(int64_t)x,mod=m->ple_head_vocab[head],r=sx%mod;if(r<0)r+=mod;+ return m->ple_head_offset[head]+r;+}+++/* Anticipa le letture PLE all'inizio del forward invece di emetterle inline al+ * layer che le usa.+ *+ * Perche' si puo' fare, e perche' NON si puo' fare con gli esperti: gli indici+ * delle righe PLE sono funzione pura degli id dei token e della finestra di due+ * token che li precede -- q38_hash_row non guarda nessuno stato nascosto. Sono+ * quindi noti PRIMA che parta un solo calcolo. Gli esperti no: quelli dipendono+ * dal router del layer precedente, e per questo la loro finestra di anticipo e'+ * di un layer e non di tutto il forward.+ *+ * Il PLE cade sul layer 2 di 48, quindi fra l'emissione e l'uso ci sono due+ * layer interi di calcolo -- con il loro streaming di esperti, che su questo+ * motore e' la parte lenta. E' abbondantemente il tempo di far arrivare 16+ * righe da 160 byte.+ *+ * Sono anche letture PARALLELE invece che in fila: prima erano 16 pread seriali+ * a queue depth 1 per token, che in prefill diventano lunghezza-del-prompt per+ * 16, una alla volta.+ *+ * Riordino puro: stessi byte, letti prima. I token non cambiano, e il test lo+ * pretende invece di darlo per scontato. */+static void q38_ple_prefetch(Model *m,const int *ids,int S) {+ Cfg *c=&m->c;+ free(m->ple_pref); m->ple_pref=NULL; m->ple_pref_rows=0;+ if(c->ple_layer<0||S<1||c->ngram_heads<1||c->ngram_head_dim<1) return;+ if(!q38_env_bool("Q38_PLE_PREFETCH",1)) return;++ int64_t per_row=c->ngram_head_dim, per_pos=(int64_t)c->ngram_heads*per_row;+ if(S>INT_MAX/c->ngram_heads) return;+ float *buffer=(float*)malloc((size_t)S*per_pos*sizeof(float));+ int64_t *rows=(int64_t*)malloc((size_t)S*c->ngram_heads*sizeof(int64_t));+ if(!buffer||!rows){free(buffer);free(rows);return;} /* niente prefetch: si legge inline */++ /* La finestra di due token viene SIMULATA, non mutata: q38_ple la aggiorna+ * per conto suo mentre gira, e toccarla qui la farebbe avanzare due volte. */+ int64_t history[2]={m->ple_history[0],m->ple_history[1]};+ int history_len=m->ple_history_len;+ for(int s=0;s<S;s++){+ int64_t p1=history_len>=1?history[history_len-1]:c->eos_id;+ int64_t p2=history_len>=2?history[history_len-2]:c->eos_id;+ for(int h=0;h<c->ngram_heads;h++){+ int ng=h<c->heads_per_ngram?2:3;+ rows[(int64_t)s*c->ngram_heads+h]=q38_hash_row(m,h,ng,ids[s],p1,p2);+ }+ if(ids[s]==c->eos_id) history_len=0;+ else if(history_len==0){history[0]=ids[s];history_len=1;}+ else if(history_len==1){history[1]=ids[s];history_len=2;}+ else {history[0]=history[1];history[1]=ids[s];}+ }++ int64_t total=(int64_t)S*c->ngram_heads;+ volatile int failed=0;+ #pragma omp parallel for schedule(static)+ for(int64_t i=0;i<total;i++){+ if(failed) continue;+ q38_ple_row(m,rows[i],buffer+i*per_row);+ }+ free(rows);+ if(failed){free(buffer);return;}+ m->ple_pref=buffer; m->ple_pref_rows=S;+}++static void q38_ple(Model *m,const int *ids,int S,const float *hyper,float *out) {+ double phase_started=now_s();+ Cfg *c=&m->c; Layer *l=&m->L[c->ple_layer]; int H=c->hidden,C=c->hc_count,W=c->hc_width,E=c->ple_dim;+ float *emb=falloc(E),*keys=falloc(W),*value=falloc(H),*kn=falloc(W),*qn=falloc(W),*gated=falloc(W),*norm=falloc(W);+ int state_len=(c->ple_convk-1)*c->ngram_size; float *ring=m->PLE_conv_state;+ for(int s=0;s<S;s++){+ int64_t p1=m->ple_history_len>=1?m->ple_history[m->ple_history_len-1]:c->eos_id;+ int64_t p2=m->ple_history_len>=2?m->ple_history[m->ple_history_len-2]:c->eos_id;+ if(m->ple_pref&&s<m->ple_pref_rows){+ /* gia' in memoria: le ha portate q38_ple_prefetch mentre i primi+ * layer calcolavano */+ memcpy(emb,m->ple_pref+(int64_t)s*c->ngram_heads*c->ngram_head_dim,+ (size_t)c->ngram_heads*c->ngram_head_dim*sizeof(float));+ } else for(int h=0;h<c->ngram_heads;h++){+ int ng=h<c->heads_per_ngram?2:3; int64_t row=q38_hash_row(m,h,ng,ids[s],p1,p2);+ q38_ple_row(m,row,emb+(int64_t)h*c->ngram_head_dim);+ }+ q38_dense_matmul(m,keys,emb,&l->ple_key,1,E,W);q38_dense_matmul(m,value,emb,&l->ple_value,1,E,H);+ for(int b=0;b<C;b++){+ q38_rms0(kn+(int64_t)b*H,keys+(int64_t)b*H,l->ple_norm_key+(int64_t)b*H,H,c->eps);+ q38_rms0(qn+(int64_t)b*H,hyper+(int64_t)s*W+(int64_t)b*H,l->ple_norm_query+(int64_t)b*H,H,c->eps);+ float dot=0.f;for(int d=0;d<H;d++)dot+=kn[(int64_t)b*H+d]*qn[(int64_t)b*H+d];dot/=sqrtf((float)H);+ float shaped=copysignf(sqrtf(fmaxf(fabsf(dot),1e-6f)),dot),g=q38_sigmoid(shaped);+ for(int d=0;d<H;d++)gated[(int64_t)b*H+d]=g*value[d];+ q38_rms0(norm+(int64_t)b*H,gated+(int64_t)b*H,l->ple_norm_conv+(int64_t)b*H,H,c->eps);+ }+ for(int d=0;d<W;d++){+ float a=l->ple_conv[(int64_t)d*c->ple_convk+c->ple_convk-1]*norm[d];+ for(int k=0;k<c->ple_convk-1;k++)a+=l->ple_conv[(int64_t)d*c->ple_convk+k]*ring[(int64_t)d*state_len+k*c->ngram_size];+ out[(int64_t)s*W+d]=gated[d]+q38_silu(a);+ float *r=ring+(int64_t)d*state_len;for(int k=0;k<state_len-1;k++)r[k]=r[k+1];r[state_len-1]=norm[d];+ }+ if(ids[s]==c->eos_id)m->ple_history_len=0;+ else if(m->ple_history_len==0){m->ple_history[0]=ids[s];m->ple_history_len=1;}+ else if(m->ple_history_len==1){m->ple_history[1]=ids[s];m->ple_history_len=2;}+ else {m->ple_history[0]=m->ple_history[1];m->ple_history[1]=ids[s];}+ }+ free(emb);free(keys);free(value);free(kn);free(qn);free(gated);free(norm);+ q38_tm_add(m,Q38_TM_PLE,phase_started);+}++/* Choose a context-independent prefill chunk whose private workspace fits the+ * common target. Callers provide exact fixed and per-row byte counts; even a+ * hostile-but-valid geometry gets one row rather than an unbounded allocation. */+static int q38_bounded_prefill_rows(int requested,uint64_t fixed,+ uint64_t per_row) {+ int rows=requested<Q38_PREFILL_BATCH_ROWS?requested:Q38_PREFILL_BATCH_ROWS;+ if(rows<1)return 1;+ for(;rows>1;rows--)+ if(per_row<=UINT64_MAX/(uint64_t)rows&&+ fixed<=UINT64_MAX-per_row*(uint64_t)rows&&+ fixed+per_row*(uint64_t)rows<=Q38_PREFILL_WORKSPACE_BYTES)+ return rows;+ return 1;+}++/* Batch the four resident DeltaNet input projections and the output projection+ * in bounded chunks. The convolution and recurrent update remain strictly+ * token-causal inside each chunk, so chunk boundaries cannot change state or+ * floating-point order. */+static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,+ float *out) {+ double phase_started=now_s();+ Cfg *c=&m->c;+ int H=c->hidden,VH=c->dn_vheads,KH=c->dn_kheads;+ int KD=c->dn_kdim,VD=c->dn_vdim,CD=c->dn_conv_dim,CK=c->dn_convk;+ int V=VH*VD,K=KH*KD,rep=VH/KH;+ uint64_t row_floats=(uint64_t)CD+2u*(uint64_t)V+2u*(uint64_t)VH;+ uint64_t fixed_floats=(uint64_t)CD+2u*(uint64_t)VH*(uint64_t)KD++ (uint64_t)V;+ uint64_t row_bytes=row_floats>UINT64_MAX/sizeof(float)?+ UINT64_MAX:row_floats*sizeof(float);+ uint64_t fixed_bytes=fixed_floats>UINT64_MAX/sizeof(float)?+ UINT64_MAX:fixed_floats*sizeof(float);+ int rows_capacity=m->prefill_batch?+ q38_bounded_prefill_rows(S,fixed_bytes,row_bytes):1;++ float *qkv=falloc((int64_t)rows_capacity*CD);+ float *z=falloc((int64_t)rows_capacity*V);+ float *bb=falloc((int64_t)rows_capacity*VH);+ float *aa=falloc((int64_t)rows_capacity*VH);+ float *norm=falloc((int64_t)rows_capacity*V);+ float *conv=falloc(CD);+ float *q=falloc((int64_t)VH*KD),*k=falloc((int64_t)VH*KD);+ float *core=falloc(V);+ float *rec=m->DN_rec[layer],*ring=m->DN_conv[layer];++ for(int base=0;base<S;) {+ int rows=S-base<rows_capacity?S-base:rows_capacity;+ const float *chunk=x+(int64_t)base*H;+ q38_dense_matmul(m,qkv,chunk,&l->dn_qkv,rows,H,CD);+ q38_dense_matmul(m,z,chunk,&l->dn_z,rows,H,V);+ q38_dense_matmul(m,bb,chunk,&l->dn_b,rows,H,VH);+ q38_dense_matmul(m,aa,chunk,&l->dn_a,rows,H,VH);++ for(int s=0;s<rows;s++) {+ float *qkv_row=qkv+(int64_t)s*CD;+ float *z_row=z+(int64_t)s*V;+ float *b_row=bb+(int64_t)s*VH;+ float *a_row=aa+(int64_t)s*VH;+ for(int d=0;d<CD;d++) {+ float value=l->dn_conv[(int64_t)d*CK+CK-1]*qkv_row[d];+ float *history=ring+(int64_t)d*(CK-1);+ for(int tap=0;tap<CK-1;tap++)+ value+=l->dn_conv[(int64_t)d*CK+tap]*history[tap];+ conv[d]=q38_silu(value);+ for(int tap=0;tap<CK-2;tap++)history[tap]=history[tap+1];+ history[CK-2]=qkv_row[d];+ }+ const float *qi=conv,*ki=conv+K,*vi=conv+2*K;+ for(int h=0;h<VH;h++) {+ float *qh=q+(int64_t)h*KD,*kh=k+(int64_t)h*KD;+ memcpy(qh,qi+(int64_t)(h/rep)*KD,(size_t)KD*sizeof(float));+ memcpy(kh,ki+(int64_t)(h/rep)*KD,(size_t)KD*sizeof(float));+ double qsum=1e-6,ksum=1e-6;+ for(int d=0;d<KD;d++) {+ qsum+=(double)qh[d]*qh[d];+ ksum+=(double)kh[d]*kh[d];+ }+ float qscale=1.f/sqrtf((float)qsum)/sqrtf((float)KD);+ float kscale=1.f/sqrtf((float)ksum);+ for(int d=0;d<KD;d++){qh[d]*=qscale;kh[d]*=kscale;}+ }+ #pragma omp parallel for schedule(static)+ for(int h=0;h<VH;h++) {+ float *state=rec+(int64_t)h*KD*VD;+ const float *qh=q+(int64_t)h*KD;+ const float *kh=k+(int64_t)h*KD;+ const float *vh=vi+(int64_t)h*VD;+ float alpha=expf(-expf(l->dn_alog[h])*+ q38_softplus(a_row[h]+l->dn_dtbias[h]));+ float beta=q38_sigmoid(b_row[h]);+ float delta[512];+ int64_t state_cells=(int64_t)KD*VD;+ for(int64_t cell=0;cell<state_cells;cell++)state[cell]*=alpha;+ for(int value=0;value<VD;value++) {+ float previous=0.f;+ for(int d=0;d<KD;d++)+ previous+=kh[d]*state[(int64_t)d*VD+value];+ delta[value]=(vh[value]-previous)*beta;+ }+ for(int d=0;d<KD;d++)for(int value=0;value<VD;value++)+ state[(int64_t)d*VD+value]+=kh[d]*delta[value];+ for(int value=0;value<VD;value++) {+ float current=0.f;+ for(int d=0;d<KD;d++)+ current+=qh[d]*state[(int64_t)d*VD+value];+ core[(int64_t)h*VD+value]=current;+ }+ }+ float *norm_row=norm+(int64_t)s*V;+ for(int h=0;h<VH;h++)+ q38_rmsg(norm_row+(int64_t)h*VD,core+(int64_t)h*VD,+ z_row+(int64_t)h*VD,l->dn_norm,VD,c->eps,1);+ }+ q38_dense_matmul(m,out+(int64_t)base*H,norm,&l->dn_out,+ rows,V,H);+ base+=rows;+ }+ free(qkv);free(z);free(bb);free(aa);free(norm);free(conv);+ free(q);free(k);free(core);+ q38_tm_add(m,Q38_TM_DELTANET,phase_started);+}++typedef struct { float score; int block; } Q38Block;+static int q38_block_desc(const void *aa,const void *bb){+ const Q38Block *a=(const Q38Block*)aa,*b=(const Q38Block*)bb;+ if(a->score>b->score)return -1;if(a->score<b->score)return 1;return a->block-b->block;+}++static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int pos_base,float *out) {+ Cfg *c=&m->c;int H=c->hidden,QH=c->q_heads,KVH=c->kv_heads,D=c->head_dim;+ int IQ=c->idx_qheads,ID=c->idx_dim,R=c->idx_ratio,maxsel=c->idx_budget+R-1;+ float *qp=falloc((int64_t)S*QH*2*D),*kp=falloc((int64_t)S*KVH*D),*vp=falloc((int64_t)S*KVH*D);+ float *ip=falloc((int64_t)S*(IQ+c->idx_kheads)*ID);+ q38_dense_matmul(m,qp,x,&l->q,S,H,QH*2*D);q38_dense_matmul(m,kp,x,&l->k,S,H,KVH*D);q38_dense_matmul(m,vp,x,&l->v,S,H,KVH*D);+ q38_dense_matmul(m,ip,x,&l->idx_qk,S,H,(IQ+c->idx_kheads)*ID);+ for(int s=0;s<S;s++){+ int pos=pos_base+s;+ for(int h=0;h<KVH;h++){+ float *kh=kp+(int64_t)s*KVH*D+(int64_t)h*D;q38_rms0(kh,kh,l->kn,D,c->eps);q38_rope(kh,D,c->rotary_dim,pos,c->theta);+ memcpy(m->K[layer]+((int64_t)h*m->kv_cap+pos)*D,kh,(size_t)D*sizeof(float));+ memcpy(m->V[layer]+((int64_t)h*m->kv_cap+pos)*D,vp+(int64_t)s*KVH*D+(int64_t)h*D,(size_t)D*sizeof(float));+ }+ memcpy(m->IK[layer]+(int64_t)pos*ID,ip+(int64_t)s*(IQ+1)*ID+(int64_t)IQ*ID,(size_t)ID*sizeof(float));+ }+ float *heads=falloc((int64_t)S*QH*D),*qidx=falloc((int64_t)IQ*ID),*pool=falloc(ID);+ int *selected=(int*)malloc((size_t)maxsel*sizeof(int));+ if(!selected){fprintf(stderr,"OOM QSA selection\n");exit(1);}+ for(int s=0;s<S;s++){+ int pos=pos_base+s,visible=pos+1,blocks=visible/R,tail=blocks*R;+ double phase_started=now_s();+ for(int h=0;h<IQ;h++){float *qh=qidx+(int64_t)h*ID;memcpy(qh,ip+(int64_t)s*(IQ+1)*ID+(int64_t)h*ID,(size_t)ID*sizeof(float));q38_rms0(qh,qh,l->idx_qn,ID,c->eps);q38_rope(qh,ID,c->rotary_dim,pos,c->theta);}+ int take=blocks<c->idx_budget/R?blocks:c->idx_budget/R,nsel=0;+ Q38Block *rank=blocks?(Q38Block*)malloc((size_t)blocks*sizeof(Q38Block)):NULL;+ if(blocks&&!rank){fprintf(stderr,"OOM QSA block ranking\n");exit(1);}+ for(int b=0;b<blocks;b++){+ memset(pool,0,(size_t)ID*sizeof(float));for(int r=0;r<R;r++){const float *raw=m->IK[layer]+(int64_t)(b*R+r)*ID;for(int d=0;d<ID;d++)pool[d]+=raw[d]/R;}+ q38_rms0(pool,pool,l->idx_kn,ID,c->eps);q38_rope(pool,ID,c->rotary_dim,b*R,c->theta);+ float score=0.f;for(int h=0;h<IQ;h++){float a=0.f;for(int d=0;d<ID;d++)a+=qidx[(int64_t)h*ID+d]*pool[d];if(a>0.f)score+=a;}rank[b]=(Q38Block){score/sqrtf((float)ID),b};+ }+ if(blocks)qsort(rank,(size_t)blocks,sizeof(Q38Block),q38_block_desc);+ for(int z=0;z<take;z++)for(int r=0;r<R;r++)selected[nsel++]=rank[z].block*R+r;+ for(int t=tail;t<visible;t++)selected[nsel++]=t;free(rank);+ q38_tm_add(m,Q38_TM_QSA_INDEX,phase_started); phase_started=now_s();+ for(int h=0;h<QH;h++){+ float *qraw=qp+(int64_t)s*QH*2*D+(int64_t)h*2*D;+ float *qh=falloc(D);memcpy(qh,qraw,(size_t)D*sizeof(float));q38_rms0(qh,qh,l->qn,D,c->eps);q38_rope(qh,D,c->rotary_dim,pos,c->theta);+ float *score=falloc(nsel);float mx=-INFINITY;+ int khidx=h/(QH/KVH);for(int j=0;j<nsel;j++){const float *kh=m->K[layer]+((int64_t)khidx*m->kv_cap+selected[j])*D;float a=0.f;for(int d=0;d<D;d++)a+=qh[d]*kh[d];a/=sqrtf((float)D);score[j]=a;if(a>mx)mx=a;}+ float den=0.f;for(int j=0;j<nsel;j++){score[j]=expf(score[j]-mx);den+=score[j];}+ float *oh=heads+(int64_t)s*QH*D+(int64_t)h*D;memset(oh,0,(size_t)D*sizeof(float));+ for(int j=0;j<nsel;j++){float a=score[j]/den;const float *vh=m->V[layer]+((int64_t)khidx*m->kv_cap+selected[j])*D;for(int d=0;d<D;d++)oh[d]+=a*vh[d];}+ for(int d=0;d<D;d++)oh[d]*=q38_sigmoid(qraw[D+d]);free(qh);free(score);+ }+ q38_tm_add(m,Q38_TM_QSA_ATTENTION,phase_started);+ }+ q38_dense_matmul(m,out,heads,&l->o,S,QH*D,H);+ free(qp);free(kp);free(vp);free(ip);free(heads);free(qidx);free(pool);free(selected);+}++/* The single-row path is intentionally kept separate from prefill. Decode is+ * the latency-sensitive steady state and should not pay for route tables or a+ * prompt-sized workspace. */+/* ---- CUDA VRAM expert tier (qwen36_tier.c, fp8 streaming mode) ------------+ * The tier keeps its own copies of hot experts in VRAM; the RAM LRU below is+ * untouched by it. qt_issue() takes the resident experts of a token's route+ * off the CPU, qt_take() adds their outputs back, and every expert the CPU+ * did compute is reported with qt_note() so the tier can promote it. The+ * tier copies what it needs during qt_note; the slot may be recycled by the+ * next token. Only native FP8 slots are reported: gate.data at the slab start+ * is what q38_bind_fp8_slot() produces, an expanded slot points elsewhere. */+static void q38_tier_note(int layer,int eid,const Slot *ex) {+ if(!qt_ready()||!ex->fp8_slab||ex->gate.data!=ex->fp8_slab)return;+ qt_note(layer,eid,(const uint8_t*)ex->gate.data,(const uint8_t*)ex->up.data,+ (const uint8_t*)ex->down.data,ex->gate.scales,ex->up.scales,ex->down.scales);+}++/* ---- dense trunk on the GPU (R7b, stage 1) ---------------------------------+ * The BF16 trunk is the largest fixed cost of a decode token here (about a+ * third), and it is bandwidth-bound. Every matrix of at least 1 MiB is+ * offered to the tier's placer by name and layer before qt_init; whatever+ * the placer accepts is quantized to int8 per row at start (scale = max|w| /+ * 127, the qwen36 dnproj/lmhead format) and uploaded once; the BF16 copy+ * stays for prefill and as fallback. Q38_TRUNK_GPU=0 keeps the trunk on the+ * CPU (parity runs against the BF16 reference). */+typedef struct { Q38Weight *w; char name[16]; int layer; } Q38TrunkItem;+static Q38TrunkItem *g_trunk; static int g_trunk_n, g_trunk_cap;+static void q38_trunk_add(Q38Weight *w,const char *name,int layer) {+ if(!w||!w->data||(w->kind!=Q38_WEIGHT_BF16&&w->kind!=Q38_WEIGHT_F32))return;+ size_t bytes=(size_t)w->rows*w->cols+(size_t)w->rows*sizeof(float);+ /* Q38_TRUNK_MIN_KB (default 1024): a round trip costs more than a tiny+ * GEMV saves; Q38_TRUNK_SKIP=name,name: leave those components on the+ * CPU (bisecting a numeric difference, or a component that does not pay) */+ static long min_kb=-1; static const char *skip;+ if(min_kb<0){ const char *e=getenv("Q38_TRUNK_MIN_KB"); min_kb=e?atol(e):1024; skip=getenv("Q38_TRUNK_SKIP"); }+ if(bytes<(size_t)min_kb*1024)return;+ if(skip&&*skip){+ size_t n=strlen(name); const char *s=skip;+ while(*s){ const char *c=strchr(s,','); size_t l=c?(size_t)(c-s):strlen(s);+ if(l==n&&!strncmp(s,name,n))return; if(!c)break; s=c+1; }+ }+ if(g_trunk_n==g_trunk_cap){+ g_trunk_cap=g_trunk_cap?2*g_trunk_cap:256;+ g_trunk=(Q38TrunkItem*)realloc(g_trunk,(size_t)g_trunk_cap*sizeof(*g_trunk));+ if(!g_trunk){fprintf(stderr,"OOM trunk table\n");exit(1);}+ }+ Q38TrunkItem *it=&g_trunk[g_trunk_n++]; it->w=w; it->layer=layer;+ snprintf(it->name,sizeof it->name,"%s",name);+ qt_trunk_offer(it->name,layer,bytes);+}+static int q38_trunk_enabled(void) {+ const char *e=getenv("Q38_TRUNK_GPU"); return !(e&&e[0]=='0'&&!e[1]);+}+/* offers, before qt_init: lm_head first (the placer takes it first), then the+ * layers in order so a partial placement is a prefix of the layers */+static void q38_trunk_offer_all(Model *m) {+ if(!q38_trunk_enabled())return;+ Cfg *c=&m->c;+ q38_trunk_add(&m->lm_head,"lmhead",0);+ for(int l=0;l<c->layers;l++){+ Layer *L=&m->L[l];+ if(c->is_attn[l]){+ q38_trunk_add(&L->q,"attnq",l); q38_trunk_add(&L->k,"attnk",l);+ q38_trunk_add(&L->v,"attnv",l); q38_trunk_add(&L->o,"attno",l);+ q38_trunk_add(&L->idx_qk,"qsaidx",l);+ } else {+ q38_trunk_add(&L->dn_qkv,"dnqkv",l); q38_trunk_add(&L->dn_z,"dnz",l);+ q38_trunk_add(&L->dn_out,"dnout",l);+ }+ q38_trunk_add(&L->attn_gr.down,"hcad",l); q38_trunk_add(&L->attn_gr.up,"hcau",l);+ q38_trunk_add(&L->attn_gr.inject,"hcai",l);+ q38_trunk_add(&L->mlp_gr.down,"hcmd",l); q38_trunk_add(&L->mlp_gr.up,"hcmu",l);+ q38_trunk_add(&L->mlp_gr.inject,"hcmi",l);+ q38_trunk_add(&L->sh_g,"shg",l); q38_trunk_add(&L->sh_u,"shu",l); q38_trunk_add(&L->sh_d,"shd",l);+ q38_trunk_add(&L->router,"router",l);+ }+}+/* int8 per row, scale = max|w| / 127 (the qwen36 dnproj/lmhead format) */+static void q38_trunk_quantize(const Q38Weight *w,int8_t **qp,float **scp) {+ int O=w->rows,I=w->cols;+ int8_t *q=(int8_t*)malloc((size_t)O*I); float *sc=(float*)malloc((size_t)O*sizeof(float));+ if(!q||!sc){fprintf(stderr,"OOM trunk quantization\n");exit(1);}+ #pragma omp parallel for schedule(static)+ for(int r=0;r<O;r++){+ float row[8192]; float *src=row; float *heap=NULL;+ if(I>8192){heap=(float*)malloc((size_t)I*sizeof(float)); src=heap;}+ q38_weight_row(w,r,src);+ float mx=0.f; for(int k=0;k<I;k++){float a=fabsf(src[k]); if(a>mx)mx=a;}+ float s=mx>0.f?mx/127.f:1.f, inv=1.f/s; sc[r]=s;+ int8_t *dst=q+(size_t)r*I;+ for(int k=0;k<I;k++){int v=(int)lrintf(src[k]*inv); if(v>127)v=127; if(v<-127)v=-127; dst[k]=(int8_t)v;}+ free(heap);+ }+ *qp=q; *scp=sc;+}+/* Q38_TRUNK_CPU_INT8=1: keep the int8 rows on the CPU instead (or as well),+ * so the quantization can be judged without a GPU (PPL, token parity) */+static void q38_trunk_cpu_int8(Model *m) {+ const char *e=getenv("Q38_TRUNK_CPU_INT8");+ if(!e||e[0]!='1'||e[1])return;+ if(!g_trunk_n) q38_trunk_offer_all(m);+ double t0=now_s(); size_t bytes=0;+ for(int i=0;i<g_trunk_n;i++){+ Q38Weight *w=g_trunk[i].w; if(w->q8)continue;+ q38_trunk_quantize(w,&w->q8,&w->q8sc); bytes+=(size_t)w->rows*w->cols;+ }+ fprintf(stderr,"[qwen38] trunk: %d matrices int8 on the CPU (%.2f GiB) in %.1fs (Q38_TRUNK_CPU_INT8)\n",+ g_trunk_n,bytes/1073741824.0,now_s()-t0);+}+/* after qt_init: quantize and upload what the placer accepted */+static void q38_trunk_place_all(Model *m) {+ (void)m;+ int placed=0; size_t placed_bytes=0; double t0=now_s();+ for(int i=0;i<g_trunk_n;i++){+ Q38TrunkItem *it=&g_trunk[i]; Q38Weight *w=it->w;+ int dev=qt_place_of(it->name,it->layer);+ if(dev==QT_PLACE_CPU)continue;+ int O=w->rows,I=w->cols;+ int8_t *q; float *sc; q38_trunk_quantize(w,&q,&sc);+ int h=qt_dense_init(q,sc,I,O,dev);+ if(h>=0&&getenv("Q38_TRUNK_SELFTEST")){+ /* DIAG: GPU int8 GEMV against the same int8 matrix on the CPU */+ float *x=(float*)malloc((size_t)I*sizeof(float)),*yg=(float*)malloc((size_t)O*sizeof(float)),*yc=(float*)malloc((size_t)O*sizeof(float));+ for(int k=0;k<I;k++)x[k]=sinf(0.37f*k)+0.1f*(k%7);+ for(int r=0;r<O;r++){double a=0; for(int k=0;k<I;k++)a+=(double)q[(size_t)r*I+k]*x[k]; yc[r]=(float)(a*sc[r]);}+ int ok=qt_dense_matmul(h,yg,x,I,O); double num=0,den=0; int worst=0;+ for(int r=0;r<O;r++){double d=yg[r]-yc[r]; num+=d*d; den+=(double)yc[r]*yc[r]; if(fabs(d)>fabs(yg[worst]-yc[worst]))worst=r;}+ fprintf(stderr,"[selftest] %-7s L%-2d [O=%d I=%d] ok=%d rel.err %.2e worst row %d gpu %.5g cpu %.5g\n",it->name,it->layer,O,I,ok,den>0?sqrt(num/den):-1.0,worst,yg[worst],yc[worst]);+ free(x);free(yg);free(yc);+ }+ free(q); free(sc);+ if(h>=0){ w->gpu=h+1; placed++; placed_bytes+=(size_t)O*I; }+ }+ if(g_trunk_n)+ fprintf(stderr,"[qtier] qwen38 trunk: %d of %d offered matrices resident as int8 (%.2f GiB) in %.1fs; the rest stays BF16 on the CPU\n",+ placed,g_trunk_n,placed_bytes/1073741824.0,now_s()-t0);+}++/* Start the tier after the model is loaded. COLI_CUDA=1 turns it on (the+ * tier reads that itself); it needs every layer's experts in native FP8 with+ * the block-scale bank resident, because the GPU kernels consume exactly the+ * checkpoint layout (e4m3 bytes + [ceil(O/128), ceil(I/128)] scales). */+static void q38_tier_start(Model *m,int cap) {+ const char *on=getenv("COLI_CUDA");+ if(!on||on[0]!='1'||on[1])return;+ Cfg *c=&m->c;+ if(!m->native_fp8){+ fprintf(stderr,"[qtier] qwen38: expert tier needs native FP8 experts (Q38_NATIVE_FP8=1); staying on the CPU\n");+ return;+ }+ for(int layer=0;layer<c->layers;layer++)+ if(!q38_prepare_expert_scale_bank(m,layer)){+ fprintf(stderr,"[qtier] qwen38: layer %d experts are not native FP8; staying on the CPU\n",layer);+ return;+ }+ q38_trunk_offer_all(m); /* sizes only; the placer decides in qt_init */+ if(qt_init_fp8(c->layers,c->experts,c->hidden,c->inter,cap,c->topk,E4M3_LUT)){+ atexit(qt_shutdown);+ fprintf(stderr,"[qtier] qwen38: fp8 expert tier on (RAM LRU %d/layer stays; experts stream to VRAM as they get hot)\n",cap);+ q38_trunk_place_all(m);+ }+}++static void q38_moe_decode(Model *m,Layer *l,int layer,const float *x,int S,float *out) {+ Cfg *c=&m->c;int H=c->hidden,E=c->experts,K=c->topk,I=c->inter,SI=c->shared_inter;+ float *logits=falloc(E),*sg=falloc(SI),*su=falloc(SI),*sh=falloc(SI),*shared=falloc(H);+ float *eg=falloc(I),*eu=falloc(I),*eh=falloc(I),*eo=falloc(H);+ for(int s=0;s<S;s++){+ const float *xs=x+(int64_t)s*H;float *ys=out+(int64_t)s*H;memset(ys,0,(size_t)H*sizeof(float));+ q38_dense_matmul(m,logits,xs,&l->router,1,H,E);float mx=logits[0];for(int e=1;e<E;e++)if(logits[e]>mx)mx=logits[e];+ double all=0;for(int e=0;e<E;e++){logits[e]=expf(logits[e]-mx);all+=logits[e];}+ int idx[Q38_MAX_TOPK];float val[Q38_MAX_TOPK];+ for(int z=0;z<K;z++){int best=-1;float bv=-1.f;for(int e=0;e<E;e++){int used=0;for(int j=0;j<z;j++)if(idx[j]==e)used=1;if(!used&&logits[e]>bv){bv=logits[e];best=e;}}idx[z]=rt_router_pick(best,z,E,layer);val[z]=logits[idx[z]];}+ double top=0;for(int z=0;z<K;z++)top+=val[z];double den=c->norm_topk?top:all;+ float route_gates[Q38_MAX_TOPK];+ for(int z=0;z<K;z++) route_gates[z]=(float)(val[z]/den);+ rt_route(layer,s,idx,route_gates,K); /* shared counts + post-normalization trace */+ /* Resident experts run on the GPU from here on (asynchronously); the+ * CPU only loads and computes the rest, in route order. */+ uint32_t qmask=qt_issue(layer,idx,K,xs);+ int cpu_idx[Q38_MAX_TOPK],cpu_rank[Q38_MAX_TOPK],cpu_n=0;+ for(int z=0;z<K;z++)if(!((qmask>>z)&1u)){cpu_idx[cpu_n]=idx[z];cpu_rank[cpu_n]=z;cpu_n++;}+ q38_prefetch_native_fp8_experts(m,layer,cpu_idx,cpu_n);+ double phase_started=now_s();+ q38_weight_matmul(sg,xs,&l->sh_g,1,H,SI);q38_weight_matmul(su,xs,&l->sh_u,1,H,SI);+ for(int j=0;j<SI;j++)sh[j]=q38_silu(sg[j])*su[j];q38_weight_matmul(shared,sh,&l->sh_d,1,SI,H);+ float gate=0.f;for(int d=0;d<H;d++)gate+=xs[d]*l->sh_gate[d];gate=q38_sigmoid(gate);+ q38_tm_add(m,Q38_TM_SHARED_EXPERT,phase_started);+ Slot *selected[Q38_MAX_TOPK];+ int loaded_batch=q38_expert_get_batch(m,layer,cpu_idx,cpu_n,selected);+ for(int i=0;i<cpu_n;i++){+ int z=cpu_rank[i];+ Slot *ex=loaded_batch?selected[i]:q38_expert_get(m,layer,cpu_idx[i]);phase_started=now_s();+ q38_weight_matmul(eg,xs,&ex->gate,1,H,I);q38_weight_matmul(eu,xs,&ex->up,1,H,I);+ for(int j=0;j<I;j++)eh[j]=q38_silu(eg[j])*eu[j];q38_weight_matmul(eo,eh,&ex->down,1,I,H);+ for(int d=0;d<H;d++)ys[d]+=route_gates[z]*eo[d];+ q38_tm_add(m,Q38_TM_ROUTED_EXPERT,phase_started);+ q38_tier_note(layer,cpu_idx[i],ex);+ }+ /* GPU experts land after the CPU ones: same values, one more group in+ * the float sum (that is the only ordering difference to a CPU run). */+ qt_take(qmask,route_gates,K,ys);+ for(int d=0;d<H;d++)ys[d]+=gate*shared[d];+ }+ rt_trace_end();+ free(logits);free(sg);free(su);free(sh);free(shared);free(eg);free(eu);free(eh);free(eo);+}++typedef struct {+ int expert;+ float gate;+} Q38RouteAssignment;++/* Pick a prefill size from a fixed workspace budget. The number of rows is+ * deliberately bounded independently of the context length: a long prompt+ * therefore reuses the same route, expert and matmul buffers one chunk at a+ * time. A single assignment needs input, gate/up activations and output;+ * these are the only buffers that scale with the number of routed rows. */+static int q38_moe_prefill_rows(const Cfg *c,int requested) {+ int64_t H=c->hidden,I=c->inter,E=c->experts,K=c->topk,SI=c->shared_inter;+ uint64_t per_assignment=(uint64_t)(2LL*((int64_t)H+I))*sizeof(float)++ sizeof(Q38RouteAssignment)+2*sizeof(int);+ uint64_t per_row=(uint64_t)E*sizeof(float)++ (uint64_t)(3LL*(int64_t)SI+H+1)*sizeof(float);+ uint64_t fixed=(uint64_t)E*(4*sizeof(int)+2*sizeof(Slot*));+ if(fixed<=UINT64_MAX-sizeof(int))fixed+=sizeof(int); /* group_offsets[E] */+ else fixed=UINT64_MAX;+ uint64_t assignment_row=(uint64_t)K>UINT64_MAX/per_assignment?+ UINT64_MAX:(uint64_t)K*per_assignment;+ uint64_t total_row=per_row>UINT64_MAX-assignment_row?+ UINT64_MAX:per_row+assignment_row;+ return q38_bounded_prefill_rows(requested,fixed,total_row);+}++/* Prefill MoE: route a bounded row chunk first, then execute each distinct+ * expert's assignments as one batched SwiGLU. Grouping is an I/O optimization+ * only. Expert outputs are placed back in assignment order and the final+ * weighted reduction still visits rank 0..top-k-1 for every row, preserving the+ * decode path's floating-point accumulation order. */+static void q38_moe_prefill(Model *m,Layer *l,int layer,const float *x,+ int S,float *out) {+ Cfg *c=&m->c;+ int H=c->hidden,E=c->experts,K=c->topk,I=c->inter,SI=c->shared_inter;+ int rows_capacity=q38_moe_prefill_rows(c,S);+ int64_t max_assign=(int64_t)rows_capacity*K;++ Q38RouteAssignment *routes=(Q38RouteAssignment*)malloc(+ (size_t)max_assign*sizeof(*routes));+ int *assignments=(int*)malloc((size_t)max_assign*sizeof(*assignments));+ int *assignment_positions=(int*)malloc((size_t)max_assign*sizeof(*assignment_positions));+ int *group_counts=(int*)calloc((size_t)E,sizeof(*group_counts));+ int *group_offsets=(int*)malloc((size_t)(E+1)*sizeof(*group_offsets));+ int *group_cursor=(int*)malloc((size_t)E*sizeof(*group_cursor));+ int *unique=(int*)malloc((size_t)E*sizeof(*unique));+ Slot **batch_slots=(Slot**)calloc((size_t)E,sizeof(*batch_slots));+ if(!routes||!assignments||!assignment_positions||!group_counts||+ !group_offsets||!group_cursor||!unique||!batch_slots){+ fprintf(stderr,"OOM Qwen3.8 MoE prefill metadata\n");exit(1);+ }++ float *logits=falloc((int64_t)rows_capacity*E);+ float *shared_gate=falloc(rows_capacity);+ float *shared_g=falloc((int64_t)rows_capacity*SI);+ float *shared_u=falloc((int64_t)rows_capacity*SI);+ float *shared_hidden=falloc((int64_t)rows_capacity*SI);+ float *shared_out=falloc((int64_t)rows_capacity*H);+ float *expert_input=falloc(max_assign*H);+ float *expert_gate=falloc(max_assign*I);+ float *expert_up=falloc(max_assign*I);+ float *routed_out=falloc(max_assign*H);++ for(int base=0;base<S;) {+ int rows=S-base<rows_capacity?S-base:rows_capacity;+ int assignment_count=rows*K;+ memset(group_counts,0,(size_t)E*sizeof(*group_counts));++ /* Route the complete chunk with one resident router matmul. Selection+ * intentionally mirrors q38_moe_decode, including rt_router_pick's+ * deterministic fallback for invalid logits. */+ q38_dense_matmul(m,logits,x+(int64_t)base*H,&l->router,rows,H,E);+ for(int s=0;s<rows;s++) {+ float *probabilities=logits+(int64_t)s*E;+ float maximum=probabilities[0];+ for(int e=1;e<E;e++)if(probabilities[e]>maximum)maximum=probabilities[e];+ double total=0.0;+ for(int e=0;e<E;e++) {+ probabilities[e]=expf(probabilities[e]-maximum);+ total+=probabilities[e];+ }+ int selected[Q38_MAX_TOPK];+ float selected_probability[Q38_MAX_TOPK];+ for(int rank=0;rank<K;rank++) {+ int best=-1;float best_value=-1.f;+ for(int e=0;e<E;e++) {+ int used=0;+ for(int previous=0;previous<rank;previous++)+ if(selected[previous]==e){used=1;break;}+ if(!used&&probabilities[e]>best_value){+ best_value=probabilities[e];best=e;+ }+ }+ selected[rank]=rt_router_pick(best,rank,E,layer);+ selected_probability[rank]=probabilities[selected[rank]];+ }+ double top=0.0;+ for(int rank=0;rank<K;rank++)top+=selected_probability[rank];+ double denominator=c->norm_topk?top:total;+ float gates[Q38_MAX_TOPK];+ for(int rank=0;rank<K;rank++) {+ gates[rank]=(float)(selected_probability[rank]/denominator);+ int assignment=s*K+rank;+ routes[assignment]=(Q38RouteAssignment){selected[rank],gates[rank]};+ group_counts[selected[rank]]++;+ }+ rt_route(layer,base+s,selected,gates,K);+ }++ /* Prefixing by expert makes every group contiguous while the inverse+ * map lets the final reduction recover the original row/rank order. */+ group_offsets[0]=0;+ int unique_count=0;+ for(int e=0;e<E;e++) {+ group_offsets[e+1]=group_offsets[e]+group_counts[e];+ if(group_counts[e])unique[unique_count++]=e;+ }+ memset(group_cursor,0,(size_t)E*sizeof(*group_cursor));+ for(int assignment=0;assignment<assignment_count;assignment++) {+ int expert=routes[assignment].expert;+ int position=group_offsets[expert]+group_cursor[expert]++;+ assignments[position]=assignment;+ assignment_positions[assignment]=position;+ }++ /* One advice range per distinct expert is enough for this chunk. */+ q38_prefetch_native_fp8_experts(m,layer,unique,unique_count);++ /* Shared expert work is independent across rows and remains resident;+ * batching it here also keeps its cost out of the routed groups. */+ double phase_started=now_s();+ q38_weight_matmul(shared_g,x+(int64_t)base*H,&l->sh_g,rows,H,SI);+ q38_weight_matmul(shared_u,x+(int64_t)base*H,&l->sh_u,rows,H,SI);+ for(int s=0;s<rows;s++)+ for(int j=0;j<SI;j++)+ shared_hidden[(int64_t)s*SI+j]=+ q38_silu(shared_g[(int64_t)s*SI+j])*+ shared_u[(int64_t)s*SI+j];+ q38_weight_matmul(shared_out,shared_hidden,&l->sh_d,rows,SI,H);+ for(int s=0;s<rows;s++) {+ const float *xs=x+(int64_t)(base+s)*H;+ float gate=0.f;+ for(int d=0;d<H;d++)gate+=xs[d]*l->sh_gate[d];+ shared_gate[s]=q38_sigmoid(gate);+ }+ q38_tm_add(m,Q38_TM_SHARED_EXPERT,phase_started);++ /* A demand-set batch is particularly effective for native FP8: reserve+ * all slots before workers read the two coalesced ranges. Other native+ * layouts use the same grouped matmul loop with the ordinary bounded+ * LRU loader, so no format loses correctness or caching. */+ memset(out+(int64_t)base*H,0,(size_t)rows*H*sizeof(float));+ /* A prompt chunk can route to more distinct experts than the retained+ * cache can hold. Load and consume cache-sized groups instead of+ * declining the complete parallel demand set: every expert is still+ * loaded once for this chunk, and a later group may safely reuse its+ * slots because the preceding outputs already live in routed_out. */+ int load_limit=m->cache[layer].cap;+ if(load_limit>Q38_MAX_TOPK)load_limit=Q38_MAX_TOPK;+ if(load_limit<1)load_limit=1;+ for(int unique_base=0;unique_base<unique_count;) {+ int load_count=unique_count-unique_base;+ if(load_count>load_limit)load_count=load_limit;+ int loaded_batch=load_count>=2&&q38_expert_get_batch(+ m,layer,unique+unique_base,load_count,batch_slots);+ for(int offset=0;offset<load_count;offset++) {+ int e=unique[unique_base+offset];+ int count=group_counts[e];+ Slot *expert=loaded_batch?batch_slots[offset]:+ q38_expert_get(m,layer,e);+ int first=group_offsets[e];+ for(int a=0;a<count;a++) {+ int assignment=assignments[first+a];+ int row=assignment/K;+ memcpy(expert_input+(int64_t)a*H,+ x+(int64_t)(base+row)*H,(size_t)H*sizeof(float));+ }+ phase_started=now_s();+ q38_weight_matmul(expert_gate,expert_input,&expert->gate,+ count,H,I);+ q38_weight_matmul(expert_up,expert_input,&expert->up,count,H,I);+ for(int a=0;a<count;a++)for(int j=0;j<I;j++)+ expert_gate[(int64_t)a*I+j]=+ q38_silu(expert_gate[(int64_t)a*I+j])*+ expert_up[(int64_t)a*I+j];+ q38_weight_matmul(routed_out+(int64_t)first*H,expert_gate,+ &expert->down,count,I,H);+ q38_tm_add(m,Q38_TM_ROUTED_EXPERT,phase_started);+ }+ unique_base+=load_count;+ }++ /* The grouped execution above is intentionally not the reduction+ * order. Replaying the original top-k sequence gives the same result+ * as the single-row implementation for F32, BF16 and block-FP8. */+ for(int s=0;s<rows;s++) {+ float *ys=out+(int64_t)(base+s)*H;+ for(int rank=0;rank<K;rank++) {+ int assignment=s*K+rank;+ const float *expert_output=routed_out++ (int64_t)assignment_positions[assignment]*H;+ float gate=routes[assignment].gate;+ for(int d=0;d<H;d++)ys[d]+=gate*expert_output[d];+ }+ const float *shared=shared_out+(int64_t)s*H;+ for(int d=0;d<H;d++)ys[d]+=shared_gate[s]*shared[d];+ }+ base+=rows;+ }+ rt_trace_end();+ free(logits);free(shared_gate);free(shared_g);free(shared_u);+ free(shared_hidden);free(shared_out);free(expert_input);free(expert_gate);+ free(expert_up);free(routed_out);free(routes);free(assignments);+ free(assignment_positions);free(group_counts);free(group_offsets);+ free(group_cursor);free(unique);free(batch_slots);+}++static void q38_moe(Model *m,Layer *l,int layer,const float *x,int S,float *out) {+ if(S<=1||!m->prefill_batch)q38_moe_decode(m,l,layer,x,S,out);+ else q38_moe_prefill(m,l,layer,x,S,out);+}++static void reset_recurrent(Model *m) {+ Cfg *c=&m->c;+ for(int i=0;i<c->layers;i++)if(!c->is_attn[i]){+ memset(m->DN_rec[i],0,(size_t)c->dn_vheads*c->dn_kdim*c->dn_vdim*sizeof(float));+ memset(m->DN_conv[i],0,(size_t)c->dn_conv_dim*(c->dn_convk-1)*sizeof(float));+ }+ memset(m->PLE_conv_state,0,(size_t)c->hc_width*(c->ple_convk-1)*c->ngram_size*sizeof(float));m->ple_history_len=0;+}++static void ensure_kv(Model *m) {+ Cfg *c=&m->c;if(m->max_t<=m->kv_cap&&m->K)return;+ if(m->K){for(int i=0;i<c->layers;i++){free(m->K[i]);free(m->V[i]);free(m->IK[i]);}free(m->K);free(m->V);free(m->IK);}+ m->K=(float**)calloc((size_t)c->layers,sizeof(float*));m->V=(float**)calloc((size_t)c->layers,sizeof(float*));m->IK=(float**)calloc((size_t)c->layers,sizeof(float*));+ for(int i=0;i<c->layers;i++)if(c->is_attn[i]){+ m->K[i]=falloc((int64_t)c->kv_heads*m->max_t*c->head_dim);m->V[i]=falloc((int64_t)c->kv_heads*m->max_t*c->head_dim);m->IK[i]=falloc((int64_t)m->max_t*c->idx_dim);+ }+ m->kv_cap=m->max_t;+}++/* Run only the requested native layer interval over hyper-residual activations.+ * Segment callers supply the four-stream boundary state directly; unlike step,+ * this path deliberately does not gather embeddings or apply the final mixer. */+static void q38_layers_forward_range(Model *m,float *hyper,const int *ids,+ int S,int pos_base,int layer_begin,+ int layer_end) {+ Cfg *c=&m->c; int H=c->hidden,W=c->hc_width,C=c->hc_count;+ float *mixed=falloc((int64_t)S*H),*inject=falloc((int64_t)S*C),*block=falloc((int64_t)S*H);+ for(int i=layer_begin;i<layer_end;i++){+ Layer *l=&m->L[i];+ if(i==c->ple_layer){+ float *ple=falloc((int64_t)S*W); q38_ple(m,ids,S,hyper,ple);+ for(int64_t z=0;z<(int64_t)S*W;z++) hyper[z]+=ple[z];+ free(ple);+ }+ q38_gr_read(m,&l->attn_gr,hyper,S,mixed,inject);+ if(c->is_attn[i]) q38_attention(m,l,i,mixed,S,pos_base,block);+ else q38_deltanet(m,l,i,mixed,S,block);+ q38_gr_apply(c,hyper,block,inject,S);+ q38_gr_read(m,&l->mlp_gr,hyper,S,mixed,inject);+ q38_moe(m,l,i,mixed,S,block);+ q38_gr_apply(c,hyper,block,inject,S);+ }+ free(mixed); free(inject); free(block);+}++/* Canale logprobs (modalita jev). Qui la fotografia dello stato NON si+ * aggiunge: questo motore ce l'ha gia, si chiama Q38PrefixCache e salva stato+ * ricorrente, id e logit finali dopo ogni prompt. Manca solo la lettura, cioe+ * un passaggio di lm_head per posizione invece che solo sull'ultima, e il+ * predittore del primo token fresco, che sono i logit gia salvati. */+static int g_echo_k = 0;+static const char *g_echo_id = NULL;+static const float *g_echo_pin_logit = NULL; /* logit dell'ultima posizione riusata */++static void q38_echo(const char *id, int pos, int token, const float *lo, int V, int k){+ char tail[1024]; coli_logprob_tail(tail, sizeof tail, lo, V, token, k);+ unsigned char *piece=NULL; int n=0;+ if(decode_id_alloc(token,&piece,&n)) { piece=NULL; n=0; }+ if(n<0) n=0;+ printf("ECHO %s %d %d%s\n", id, n, pos, tail);+ if(n>0) fwrite(piece,1,(size_t)n,stdout);+ fputc('\n', stdout); fflush(stdout);+ free(piece);+}++static float *step(Model *m,const int *ids,int S,int pos_base) {+ Cfg *c=&m->c;int H=c->hidden,W=c->hc_width,C=c->hc_count;+ m->timers.forwards++;+ float *hyper=falloc((int64_t)S*W);+ /* Le righe PLE partono ADESSO, non al layer 2 dove servono: sono note dagli+ * id dei token soltanto, e i due layer che le precedono danno il tempo di+ * farle arrivare dal disco. */+ q38_ple_prefetch(m,ids,S);+ for(int s=0;s<S;s++){+ if(ids[s]<0||ids[s]>=c->vocab){fprintf(stderr,"token id %d outside vocabulary\n",ids[s]);exit(1);}+ float *e=hyper+(int64_t)s*W;+ int abs_pos=pos_base+s, vis_row=-1;+ if(m->vis_map&&abs_pos>=0&&abs_pos<m->vis_map_len) vis_row=m->vis_map[abs_pos];+ if(vis_row>=0&&vis_row<m->vis_rows_n)+ memcpy(e,m->vis_rows+(int64_t)vis_row*H,(size_t)H*sizeof(float));+ else q38_weight_row(&m->embed,ids[s],e);+ for(int b=1;b<C;b++)memcpy(e+(int64_t)b*H,e,(size_t)H*sizeof(float));+ }+ float *mixed=falloc((int64_t)S*H),*inject=falloc((int64_t)S*C),*block=falloc((int64_t)S*H);+ for(int i=0;i<c->layers;i++){+ Layer *l=&m->L[i];+ if(i==c->ple_layer){float *ple=falloc((int64_t)S*W);q38_ple(m,ids,S,hyper,ple);for(int64_t z=0;z<(int64_t)S*W;z++)hyper[z]+=ple[z];free(ple);+ /* consumate: il chunk successivo ha altri token, e riusare queste+ * righe darebbe gli embedding del chunk precedente combaciando in+ * silenzio invece di dare errore. */+ free(m->ple_pref); m->ple_pref=NULL; m->ple_pref_rows=0;}+ q38_gr_read(m,&l->attn_gr,hyper,S,mixed,inject);+ if(c->is_attn[i])q38_attention(m,l,i,mixed,S,pos_base,block);else q38_deltanet(m,l,i,mixed,S,block);+ q38_gr_apply(c,hyper,block,inject,S);+ q38_gr_read(m,&l->mlp_gr,hyper,S,mixed,inject);q38_moe(m,l,i,mixed,S,block);q38_gr_apply(c,hyper,block,inject,S);+ }+ q38_gr_read(m,&m->final_gr,hyper,S,mixed,NULL);m->kv_len=pos_base+S;+ float *logit=falloc(c->vocab);double phase_started=now_s();+ /* Lettura del prefill: la posizione p predice il token p+1. Il primo token+ * fresco lo predice la fotografia del prefisso, quando c'e. Pagata solo da+ * chi ha chiesto il canale. */+ if(g_echo_k>0&&g_echo_id&&S>0){+ if(g_echo_pin_logit)+ q38_echo(g_echo_id,pos_base,ids[0],g_echo_pin_logit,c->vocab,g_echo_k);+ for(int p=0;p+1<S;p++){+ q38_weight_matmul(logit,mixed+(int64_t)p*H,&m->lm_head,1,H,c->vocab);+ q38_echo(g_echo_id,pos_base+p+1,ids[p+1],logit,c->vocab,g_echo_k);+ }+ }+ q38_weight_matmul(logit,mixed+(int64_t)(S-1)*H,&m->lm_head,1,H,c->vocab);+ q38_tm_add(m,Q38_TM_LM_HEAD,phase_started);+ free(hyper);free(mixed);free(inject);free(block);return logit;+}++static int q38_tm_enabled(void) {+ const char *enabled=getenv("COLI_TIMERS");+ return enabled&&enabled[0]=='1'&&enabled[1]=='\0';+}++static void q38_tm_report_bank(const Q38Timers *timers,const char *scope) {+ if(!q38_tm_enabled())return;+ static const char *names[Q38_TM_COUNT]={+ "expert-read","fp8-expand","routed-expert","shared-expert",+ "resident-mm","deltanet","qsa-index","qsa-attn","ple","lm-head"+ };+ double per=timers->forwards?1000.0/timers->forwards:0.0;+ fprintf(stderr,"[qwen38 timers] %s: %llu forwards\n",scope,+ (unsigned long long)timers->forwards);+ for(int i=0;i<Q38_TM_COUNT;i++)+ fprintf(stderr,"[qwen38 timers] %-14s %9.3f s %9.3f ms/forward\n",+ names[i],timers->seconds[i],timers->seconds[i]*per);+ fprintf(stderr,"[qwen38 timers] architecture phases overlap resident-mm; "+ "expert-read is disk service and fp8-expand is synchronous miss work\n");+}++/* Snapshot of the timer bank taken when the prompt's forward has finished+ * (generate() sets it), so a "decode" bank can be reported apart from the+ * prefill. Without it the per-forward figures of a long prompt are prefill+ * work divided by decode forwards, which reads like a decode budget and is+ * not one. */+static Q38Timers g_tm_prefill_snapshot; static int g_tm_have_snapshot;+static void q38_tm_snapshot_prefill(const Model *m) {+ g_tm_prefill_snapshot=m->timers; g_tm_have_snapshot=1;+}+static void tm_report(const Model *m) {+ q38_tm_report_bank(&m->timers,"total");+ if(g_tm_have_snapshot&&q38_tm_enabled()){+ Q38Timers decode=m->timers;+ for(int i=0;i<Q38_TM_COUNT;i++)decode.seconds[i]-=g_tm_prefill_snapshot.seconds[i];+ decode.forwards-=g_tm_prefill_snapshot.forwards;+ q38_tm_report_bank(&g_tm_prefill_snapshot,"prefill");+ q38_tm_report_bank(&decode,"decode");+ }+ if(q38_tm_enabled())+ fprintf(stderr,"[qwen38 expert I/O] weight-ranges=%llu scale-ranges=%llu "+ "coalesced-gate-up=%llu prefetched=%llu parallel-batches=%llu "+ "resident-scales=%.2f MiB\n",+ (unsigned long long)m->expert_weight_reads,+ (unsigned long long)m->expert_scale_reads,+ (unsigned long long)m->expert_pair_reads,+ (unsigned long long)m->expert_prefetch_ranges,+ (unsigned long long)m->expert_parallel_batches,+ m->expert_scale_bytes/1048576.0);+}++static void q38_layer_free(Layer *l) {+ if(!l) return;+ free(l->attn_gr.norm);q38_weight_free(&l->attn_gr.down);q38_weight_free(&l->attn_gr.up);q38_weight_free(&l->attn_gr.inject);+ free(l->mlp_gr.norm);q38_weight_free(&l->mlp_gr.down);q38_weight_free(&l->mlp_gr.up);q38_weight_free(&l->mlp_gr.inject);+ q38_weight_free(&l->router);q38_weight_free(&l->sh_g);q38_weight_free(&l->sh_u);q38_weight_free(&l->sh_d);free(l->sh_gate);+ q38_weight_free(&l->q);q38_weight_free(&l->k);q38_weight_free(&l->v);q38_weight_free(&l->o);free(l->qn);free(l->kn);+ q38_weight_free(&l->idx_qk);free(l->idx_qn);free(l->idx_kn);+ q38_weight_free(&l->dn_qkv);q38_weight_free(&l->dn_z);q38_weight_free(&l->dn_b);q38_weight_free(&l->dn_a);free(l->dn_conv);+ free(l->dn_dtbias);free(l->dn_alog);free(l->dn_norm);q38_weight_free(&l->dn_out);+ q38_weight_free(&l->ple_key);q38_weight_free(&l->ple_value);free(l->ple_norm_key);free(l->ple_norm_query);+ free(l->ple_norm_conv); free(l->ple_conv);+ memset(l,0,sizeof(*l));+}++static void q38_model_free(Model *m) {+ if(!m) return;+ for(int i=0;i<m->c.layers;i++) {+ if(m->L)q38_layer_free(&m->L[i]);+ if(m->cache) {+ if(m->cache[i].slots) {+ for(int s=0;s<m->cache[i].n;s++) {+ q38_weight_free(&m->cache[i].slots[s].gate);+ q38_weight_free(&m->cache[i].slots[s].up);+ q38_weight_free(&m->cache[i].slots[s].down);+ free(m->cache[i].slots[s].fp8_slab);+ }+ }+ free(m->cache[i].slots); free(m->cache[i].by_expert);+ }+ if(m->expert_scales)free(m->expert_scales[i].values);+ free(m->DN_rec ? m->DN_rec[i] : NULL); free(m->DN_conv ? m->DN_conv[i] : NULL);+ free(m->K ? m->K[i] : NULL); free(m->V ? m->V[i] : NULL); free(m->IK ? m->IK[i] : NULL);+ }+ free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);+ q38_weight_free(&m->embed);q38_weight_free(&m->lm_head);+ free(m->final_gr.norm);q38_weight_free(&m->final_gr.down);q38_weight_free(&m->final_gr.up);q38_weight_free(&m->final_gr.inject);+ free(m->ple_history); free(m->PLE_conv_state); free(m->c.is_attn); st_destroy(&m->S);+ memset(m,0,sizeof(*m));+}++#endif /* COLI_QWEN38_CORE_H */