Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen38: routed experts as int4-g64 from an experts-int4g64/ sidecar
JustVuggaa3db0acf43a
diff --git a/.gitignore b/.gitignoreindex ff65ffa1..4fbe610a 100644--- a/.gitignore+++ b/.gitignore@@ -158,6 +158,7 @@ c/kimi_k3_tiny/.coli_usagec/qwen38_vision_tiny/c/qwen38_mm_tiny/c/qwen38_tiny_fp8/+c/qwen38_tiny_int4/# MiMo tiny fixture: config.json and ref.json are committed, the rest is regeneratedc/mimo_tiny/*.safetensorsc/mimo_tiny/model.safetensors.index.jsondiff --git a/c/Makefile b/c/Makefileindex 468f3504..38ca6559 100644--- a/c/Makefile+++ b/c/Makefile@@ -1333,6 +1333,60 @@ qwen38-tiny-fp8-check: qwen38-tiny-fp8-generate qwen38$(EXE)qwen38-tier-engine-check: qwen38-tiny-fp8-generate tests/test_qwen38_tier_engine$(EXE)./tests/test_qwen38_tier_engine$(EXE)+# The routed experts as int4-g64: tools/convert_qwen38_experts_int4.py writes+# the experts-int4g64/ sidecar next to the FP8 shards and qwen38 picks it up.+# The fixture is the FP8 one with its experts scaled by 3 (at the default+# initialization zeroing every expert moves the logits less than the oracle's+# tolerance), converted; ref_int4.json is the float32 reference of the+# dequantized sidecar. Gates:+# (a) the int4 matmul equals a dequantized f32 reference, and every expert the+# single and the batch path load holds the sidecar's bytes+# (tests/test_qwen38_int4_experts);+# (b) the converter's round trip against the FP8 source, per matrix, and the+# engine refusing a sidecar its index does not describe+# (tests/test_qwen38_int4_convert.py);+# (c) the engine on the sidecar reproduces ref_int4.json through the single,+# batched, prefill-batch and mapped paths at caps 1, 2 and 4, with the+# last logits identical to the bit across them; the FP8 experts still+# reproduce ref.json and not ref_int4.json, so the gate tells them apart;+# (d) under COLI_CUDA=1 on the fake backend the tier declines the int4+# experts (it streams FP8 only) and the CPU still reproduces ref_int4.json.+.PHONY: qwen38-tiny-int4-generate qwen38-tiny-int4-check+qwen38-tiny-int4-generate:+ $(PYTHON) tools/make_qwen38_tiny.py --out ./qwen38_tiny_int4 --fp8-experts --int4-experts --expert-gain 3++qwen38-tiny-int4-check: qwen38-tiny-int4-generate qwen38-tiny-fp8-generate qwen38$(EXE) \+ tests/test_qwen38_int4_experts$(EXE) tests/test_qwen38_tier_engine$(EXE)+ OMP_NUM_THREADS=2 ./tests/test_qwen38_int4_experts$(EXE) ./qwen38_tiny_int4+ $(PYTHON) tests/test_qwen38_int4_convert.py --model ./qwen38_tiny_int4 --engine ./qwen38$(EXE)+ @d=./qwen38_tiny_int4/logits; rm -rf $$d; mkdir -p $$d; n=0; \+ for trunk in 1024 0; do for cap in 1 2 4; do for mode in single batched prefill prefill-serial mapped; do \+ case $$mode in \+ single) e="Q38_PREFILL_BATCH=0 Q38_EXPERT_PARALLEL_READS=0";; \+ batched) e="Q38_PREFILL_BATCH=0";; \+ prefill) e="Q38_PREFILL_BATCH=1";; \+ prefill-serial) e="Q38_PREFILL_BATCH=1 Q38_EXPERT_PARALLEL_READS=0";; \+ mapped) e="COLI_MAP_EXPERTS=1";; \+ esac; \+ env $$e Q38_EXPERT_INT4=1 Q38_TRUNK_MIN_KB=$$trunk DUMP=$$d/t$$trunk-c$$cap-$$mode.f32 OMP_NUM_THREADS=2 \+ SNAP=./qwen38_tiny_int4 ./qwen38$(EXE) $$cap 8 ./qwen38_tiny_int4/ref_int4.json > $$d/run.log 2>&1 \+ || { cat $$d/run.log; echo "FAIL int4 experts, trunk=$$trunk cap=$$cap $$mode"; exit 1; }; \+ n=$$((n+1)); \+ done; done; \+ for f in $$d/t$$trunk-*.f32; do cmp -s $$f $$d/t$$trunk-c1-single.f32 \+ || { echo "FAIL $$f: last logits differ from the single path"; exit 1; }; done; done; \+ for trunk in 1024 0; do \+ Q38_EXPERT_INT4=0 Q38_TRUNK_MIN_KB=$$trunk OMP_NUM_THREADS=2 SNAP=./qwen38_tiny_int4 \+ ./qwen38$(EXE) 2 8 ./qwen38_tiny_int4/ref.json > $$d/run.log 2>&1 \+ || { cat $$d/run.log; echo "FAIL FP8 experts of the int4 fixture, trunk=$$trunk"; exit 1; }; \+ if Q38_EXPERT_INT4=0 Q38_TRUNK_MIN_KB=$$trunk OMP_NUM_THREADS=2 SNAP=./qwen38_tiny_int4 \+ ./qwen38$(EXE) 2 8 ./qwen38_tiny_int4/ref_int4.json > $$d/run.log 2>&1 \+ || ! grep -q '^Matching tokens' $$d/run.log; then \+ cat $$d/run.log; echo "FAIL the FP8 experts pass the int4 reference: the gate cannot tell them apart"; exit 1; fi; \+ done; \+ echo "qwen38 int4-g64 experts: ref_int4.json reproduced in $$n configurations, last logits identical across paths; FP8 control ok"+ ./tests/test_qwen38_tier_engine$(EXE) ./qwen38_tiny_int4+# Context-size gates: KV layout, growth across requests, and the attention# capacity. Includes qwen36.c directly, so no model file is needed.# Same tier objects as the engine: the test includes qwen36.c, so with CUDA=1@@ -1553,6 +1607,11 @@ tests/test_dsv41_untrusted_load$(EXE): tests/test_dsv41_untrusted_load.c deepseetests/test_qwen38_idot$(EXE): tests/test_qwen38_idot.c qwen38.c $(VK_OBJ)$(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)+# The int4-g64 expert matmul with no model (part of test-c); given a converted+# snapshot on argv it checks the sidecar's loaders too (qwen38-tiny-int4-check).+tests/test_qwen38_int4_experts$(EXE): tests/test_qwen38_int4_experts.c qwen38.c $(VK_OBJ)+ $(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)+tests/test_qwen38_serve_framing$(EXE): tests/test_qwen38_serve_framing.c qwen38.c $(VK_OBJ)$(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)diff --git a/c/qwen38.c b/c/qwen38.cindex d7802e8a..e6343e9b 100644--- a/c/qwen38.c+++ b/c/qwen38.c@@ -1838,6 +1838,8 @@ static int q38_reference_mode(const char *path,int serve_mode){return length>=5&&!strcmp(path+length-5,".json");}+static void q38_expert_report(Model *m, int cap); /* below, beside the expert layout accounting */+#ifndef QWEN38_TEST_SERVEint main(int argc, char **argv) {/* Physical-core team sizing, as colibri/inkling/kimi_k3/olmoe/deepseek-v41@@ -1953,8 +1955,10 @@ int main(int argc, char **argv) {}Model m; model_init(&m, snap, cap, bits);+ q38_expert_int4_attach(&m, snap); /* <snap>/experts-int4g64/ when present (Q38_EXPERT_INT4) */q38_tier_start(&m, cap); /* COLI_CUDA=1: hot experts stream to VRAM (qwen36_tier.c) */q38_trunk_cpu_int8(&m); /* the trunk's int8 rows on the CPU, BF16 released (Q38_TRUNK_CPU_INT8=0 keeps BF16) */+ q38_expert_report(&m, cap); /* expert format, bytes per expert, what the cache costs */#ifdef COLI_VULKAN/* After the trunk is int8: those rows upload at their first matmul. No* device (or COLI_VULKAN unset) leaves g_vk_ready 0, the CPU path. */@@ -2248,6 +2252,52 @@ static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,*numeric_kinds=kinds;return 0;}+/* Said once at startup: the representation the routed-expert cache holds,+ * what one expert costs in it, what the cache costs full at this cap, and+ * how many experts per layer the RAM still available now (resident weights+ * loaded, cache empty) would hold at that size -- the number a cap is chosen+ * against, and the one the int4-g64 sidecar moves. */+static void q38_expert_report(Model *m, int cap) {+ Cfg *c = &m->c;+ int layers = m->range_end - m->range_begin;+ uint64_t per_capacity = 0, fixed = 0, snapshot_expert = 0;+ unsigned kinds = 0;+ if (layers < 1) return;+ if (!q38_segment_expert_layout(m, (uint32_t)m->range_begin, (uint32_t)m->range_end,+ &per_capacity, &fixed, &kinds))+ snapshot_expert = per_capacity / (uint64_t)layers;+ const char *what = kinds == Q38_EXPERT_FP8_BLOCK ? "native FP8" :+ kinds == Q38_EXPERT_FP8_EXPANDED ? "FP8 expanded to f32 (Q38_NATIVE_FP8=0)" :+ kinds == Q38_EXPERT_BF16 ? (m->native_bf16 ? "BF16" : "BF16 expanded to f32") :+ kinds == Q38_EXPERT_F16 || kinds == Q38_EXPERT_F32 ? "f32" : "mixed";+ uint64_t per_expert = snapshot_expert;+ char extra[160] = "";+ if (m->x4) {+ per_expert = (uint64_t)m->x4->record_bytes;+ snprintf(extra, sizeof extra, " (the snapshot's %s: %llu bytes)", what,+ (unsigned long long)snapshot_expert);+ what = "int4-g64 from " Q38_INT4_DIR "/"; /* and no FP8 scale bank is built */+ } else if (fixed) {+ snprintf(extra, sizeof extra, " + %.1f MiB resident scale bank", fixed / 1048576.0);+ }+ if (!per_expert) {+ fprintf(stderr, "[qwen38] routed experts: %s, size unknown; cache %d/layer\n", what, cap);+ return;+ }+ double full = (double)per_expert * (double)cap * (double)layers;+ fprintf(stderr, "[qwen38] routed experts: %s, %llu bytes (%.2f MiB) per expert%s; "+ "cache %d/layer x %d layers = %.2f GiB full",+ what, (unsigned long long)per_expert, per_expert / 1048576.0, extra,+ cap, layers, full / 1073741824.0);+ double available = compat_mem_available_gb() * 1e9;+ if (available > 0) {+ double room = available / ((double)per_expert * (double)layers);+ int holds = room > (double)c->experts ? c->experts : (int)room;+ fprintf(stderr, "; %.1f GB available now holds %d/layer", available / 1e9, holds);+ }+ fprintf(stderr, "\n");+}+static int q38_segment_cache_capacity(uint64_t bytes_per_capacity,uint64_t fixed_scale_bytes,int experts,uint64_t memory_limit,int *capacity){diff --git a/c/qwen38_core.h b/c/qwen38_core.hindex 31a63c05..3026f336 100644--- a/c/qwen38_core.h+++ b/c/qwen38_core.h@@ -10,6 +10,7 @@#ifndef COLI_QWEN38_CORE_H#define COLI_QWEN38_CORE_H#include "kv_prefix.h"+#include "expert_ffn.h" /* the int4-g64 routed experts (experts-int4g64/ sidecar) run through its f32 kernel */#include <pthread.h> /* q38_ehit_mark publishes the lazy HITS table under a lock */#ifdef COLI_VULKAN#include "backend_vulkan.h" /* COLI_VULKAN=1: the int8 trunk on a Vulkan device */@@ -44,12 +45,17 @@ typedef enum {Q38_WEIGHT_NONE = 0,Q38_WEIGHT_F32,Q38_WEIGHT_BF16,- Q38_WEIGHT_FP8+ Q38_WEIGHT_FP8,+ Q38_WEIGHT_INT4G64 /* routed experts from the experts-int4g64/ sidecar */} Q38WeightKind;+/* int4-g64 rows: ceil(cols/2) bytes of codes and one f32 scale per 64 inputs */+static inline int64_t q38_int4_row_bytes(int cols){ return ((int64_t)cols+1)/2; }+static inline int64_t q38_int4_groups(int cols){ return ((int64_t)cols+XF_BLOCK-1)/XF_BLOCK; }+typedef struct {void *data;- float *scales; /* block-FP8 only */+ float *scales; /* block-FP8: per 128x128 block; int4-g64: per row and 64 inputs */int rows, cols;int64_t elements, scale_count;Q38WeightKind kind;@@ -108,6 +114,8 @@ typedef struct {uint64_t used;void *fp8_slab;int64_t fp8_slab_bytes;+ void *int4_slab; /* one experts-int4g64/ record: codes, then scales */+ int64_t int4_slab_bytes;} Slot;typedef struct { Slot *slots; int *by_expert, n, cap; } LCache;@@ -117,6 +125,16 @@ typedef struct {int ready; /* 0 unknown, 1 resident, -1 incompatible */} Q38ExpertScaleCache;+/* The routed experts as int4-g64 (q38_expert_int4_attach): where every+ * expert's record sits in the sidecar's layer files, and its geometry. */+typedef struct {+ shards S; /* <snap>/experts-int4g64/, one file per layer */+ int *fd; /* [layer] the file holding that layer's records */+ int64_t *off; /* [layer*experts + expert] where the record starts */+ int64_t code_bytes[3], scale_count[3]; /* gate, up, down */+ int64_t scale_off, record_bytes;+} Q38Int4Experts;+typedef enum {Q38_EXPERT_BATCH_FALLBACK_NONE = 0,Q38_EXPERT_BATCH_FALLBACK_DISABLED,@@ -136,6 +154,7 @@ typedef struct {LCache *cache;uint8_t **ehit; /* experts routed this turn, for HITS (dashboard Brain) */Q38ExpertScaleCache *expert_scales;+ Q38Int4Experts *x4; /* routed experts from experts-int4g64/, NULL = the snapshot's own */uint64_t clock, hits, miss;uint64_t expert_weight_reads, expert_scale_reads, expert_pair_reads;uint64_t expert_prefetch_ranges, expert_parallel_batches;@@ -263,6 +282,9 @@ static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,int}static uint64_t q38_weight_bytes(const Q38Weight *weight) {+ if(weight->kind==Q38_WEIGHT_INT4G64)+ return (uint64_t)weight->rows*(uint64_t)q38_int4_row_bytes(weight->cols)++ (uint64_t)weight->scale_count*sizeof(float);uint64_t element_size=weight->kind==Q38_WEIGHT_F32?sizeof(float):weight->kind==Q38_WEIGHT_BF16?sizeof(uint16_t):weight->kind==Q38_WEIGHT_FP8?sizeof(uint8_t):0;@@ -355,6 +377,35 @@ static void q38_matmul_fp8(float *y,const float *x,const uint8_t *q8,const floatmatmul_fp8(y,x,q8,bscale,S,I,O);}+/* The int4-g64 routed experts (q38_expert_int4_attach) through expert_ffn.h's+ * f32 kernel: every code is exact in f32, one fma per element and one per+ * block for its scale, and the activations stay f32 as on the FP8 path. A row+ * is planar blocks of 64; a width that is not a multiple of 64 (only the tiny+ * fixtures) keeps its tail in pairs and finishes with the last group's scale.+ * Each output row is computed the same way whatever S is, so decode and the+ * prefill batch give the same bits. */+static void q38_matmul_int4g64(float *y,const float *x,const uint8_t *codes,+ const float *scales,int S,int I,int O) {+ int64_t row_bytes=q38_int4_row_bytes(I),groups=q38_int4_groups(I);+ int body=I/XF_BLOCK*XF_BLOCK;+ #pragma omp parallel for schedule(static)+ for(int o=0;o<O;o++){+ const uint8_t *w=codes+(int64_t)o*row_bytes;+ const float *sc=scales+(int64_t)o*groups;+ for(int s=0;s<S;s++){+ const float *xs=x+(int64_t)s*I;+ float a=body?xf_dot_f32(w,sc,xs,body):0.f;+ if(body<I){+ float tail=0.f;+ for(int i=body;i<I;i++)+ tail+=(float)((int)((w[i>>1]>>((i&1)*4))&15)-8)*xs[i];+ a=fmaf(tail,sc[groups-1],a);+ }+ y[(int64_t)s*O+o]=a;+ }+ }+}+#ifdef COLI_VULKAN/* COLI_VULKAN=1: a resident matrix answers from the Vulkan device, decode and* prefill alike, with the weights the CPU would read and in the order@@ -436,6 +487,8 @@ static void q38_weight_matmul(float *y,const float *x,const Q38Weight *weight,q38_matmul_bf16(y,x,(const uint16_t*)weight->data,S,I,O);else if(weight->kind==Q38_WEIGHT_FP8&&weight->scales)q38_matmul_fp8(y,x,(const uint8_t*)weight->data,weight->scales,S,I,O);+ else if(weight->kind==Q38_WEIGHT_INT4G64&&weight->scales)+ q38_matmul_int4g64(y,x,(const uint8_t*)weight->data,weight->scales,S,I,O);else {fprintf(stderr,"unsupported matmul weight kind %d\n",(int)weight->kind);exit(1);}}@@ -1298,11 +1351,22 @@ static int q38_try_load_native_fp8_expert(Model *m,int layer,int expert,Slot *slreturn 1;}-static void q38_prefetch_native_fp8_experts(Model *m,int layer,- const int *experts,int count) {- if(!m->expert_prefetch||!m->native_fp8||!experts||count<1||- !q38_prepare_expert_scale_bank(m,layer))return;+/* Advise the kernel of the experts a route is about to miss: the int4+ * sidecar's one record each, or the native FP8 pair and down ranges. */+static void q38_prefetch_experts(Model *m,int layer,const int *experts,int count) {+ if(!m->expert_prefetch||!experts||count<1)return;LCache *cache=&m->cache[layer];+ if(m->x4){+ for(int index=0;index<count;index++){+ int expert=experts[index];+ if(expert<0||expert>=m->c.experts||cache->by_expert[expert]>=0)continue;+ posix_fadvise(m->x4->fd[layer],m->x4->off[(int64_t)layer*m->c.experts+expert],+ m->x4->record_bytes,POSIX_FADV_WILLNEED);+ m->expert_prefetch_ranges++;+ }+ return;+ }+ if(!m->native_fp8||!q38_prepare_expert_scale_bank(m,layer))return;for(int index=0;index<count;index++){int expert=experts[index];st_tensor *weight[3];if(expert<0||expert>=m->c.experts||cache->by_expert[expert]>=0||@@ -1381,8 +1445,211 @@ static void q38_load_expert_slice(Model *m,const char *name,const st_tensor *tenq38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;}+/* ---- routed experts from the int4-g64 sidecar -------------------------------+ * tools/convert_qwen38_experts_int4.py writes <snap>/experts-int4g64/: one+ * safetensors file per layer and an index.json that says what is in them.+ * The release's FP8 shards are only read by it and stay usable as they are.+ * Every expert is one contiguous record, so a miss is one read:+ *+ * gate codes [I][H/2] | up codes [I][H/2] | down codes [H][I/2] |+ * gate scales [I][H/64] | up scales [I][H/64] | down scales [H][I/64]+ *+ * named like the release's tensors (...experts.E.gate_proj.weight, and the+ * same name + ".qs" for its f32 scales). The codes are expert_ffn.h's planar+ * int4 (a row is blocks of 64, byte k holding element k in its low nibble+ * and element k+32 in its high one, both as v+8; a width that is not a+ * multiple of 64 keeps its tail in pairs, as idot.h's planarize_i4 leaves+ * it), one scale per 64 inputs, quantized as tools/convert_qwen36.py --ebits+ * 4 --gs 64 does. On the release an expert is 2.76 MB, 56% of its 4.92 MB+ * in FP8: the same RAM holds 1.78x the experts and a miss reads 56% of the+ * bytes.+ * Attached by the CLI/serve engine only: the Segment and Edge adapters keep+ * the snapshot's experts, the representation their numeric class names. */+#define Q38_INT4_DIR "experts-int4g64"+#define Q38_INT4_FORMAT "colibri.qwen38.experts-int4g64"+#define Q38_INT4_VERSION 1++/* Q38_EXPERT_INT4: unset = the sidecar when there is one, 0 = never (the+ * snapshot's FP8), 1 = refuse to start without it (a benchmark that must not+ * quietly measure FP8). */+static int q38_expert_int4_wanted(void) {+ const char *value=getenv("Q38_EXPERT_INT4");+ if(!value||!*value)return -1;+ if(value[0]=='0'&&!value[1])return 0;+ if(value[0]=='1'&&!value[1])return 1;+ fprintf(stderr,"Q38_EXPERT_INT4 must be exactly 0 or 1\n");exit(1);+}++static void q38_int4_refuse(const char *dir,const char *why) {+ fprintf(stderr,"[qwen38] %s: %s -- refusing (Q38_EXPERT_INT4=0 ignores the sidecar)\n",dir,why);+ exit(1);+}++static int64_t q38_json_int(jval *object,const char *key) {+ jval *value=json_get(object,key);+ if(!value||value->t!=J_NUM||!isfinite(value->num)||floor(value->num)!=value->num||+ value->num<0||value->num>(double)INT64_MAX/2)return -1;+ return (int64_t)value->num;+}++/* The six tensors of one expert must be the record the index describes:+ * right dtype and shape, one file, back to back in record order, at an+ * offset that keeps the f32 scales aligned in a mapping. */+static int q38_int4_record_at(Model *m,Q38Int4Experts *x,int layer,int expert,+ int *fd,int64_t *start) {+ Cfg *c=&m->c;const char *projection[3]={"gate_proj","up_proj","down_proj"};+ int rows[3]={c->inter,c->inter,c->hidden},cols[3]={c->hidden,c->hidden,c->inter};+ char suffix[192],name[320];int64_t at=0;+ for(int part=0;part<6;part++){+ int k=part%3,is_scale=part>=3;+ int length=snprintf(suffix,sizeof suffix,"mlp.experts.%d.%s.weight%s",+ expert,projection[k],is_scale?".qs":"");+ if(length<0||(size_t)length>=sizeof suffix)return -1;+ q38_name(m,name,sizeof name,layer,suffix);+ st_tensor *t=st_find(&x->S,name);+ int64_t width=is_scale?q38_int4_groups(cols[k]):q38_int4_row_bytes(cols[k]);+ int64_t bytes=is_scale?x->scale_count[k]*(int64_t)sizeof(float):x->code_bytes[k];+ if(!t||t->dtype!=(is_scale?2:3)||t->rank!=2||t->shape[0]!=rows[k]||+ t->shape[1]!=width||t->nbytes!=bytes)return -1;+ if(!part){+ if(t->off%(int64_t)sizeof(float))return -1;+ *fd=t->fd;*start=t->off;+ }else if(t->fd!=*fd||t->off!=*start+at)return -1;+ at+=bytes;+ }+ return 0;+}++/* Read <snap>/experts-int4g64/index.json and, if it describes a complete+ * conversion of this model, index every record of the loaded layers. A+ * sidecar for another geometry or format, or whose files disagree with the+ * index, is refused rather than ignored; one still being written is skipped+ * with a line saying so (or refused under Q38_EXPERT_INT4=1). */+static void q38_expert_int4_attach(Model *m,const char *snap) {+ int wanted=q38_expert_int4_wanted();+ if(!wanted)return;+ Cfg *c=&m->c;char dir[2100],path[2200];+ snprintf(dir,sizeof dir,"%s/%s",snap,Q38_INT4_DIR);+ snprintf(path,sizeof path,"%s/index.json",dir);+ FILE *f=fopen(path,"rb");+ if(!f){+ if(wanted>0){+ fprintf(stderr,"Q38_EXPERT_INT4=1 but %s cannot be opened "+ "(tools/convert_qwen38_experts_int4.py writes it)\n",path);exit(1);+ }+ return;+ }+ char *text=NULL,*arena=NULL;long size=-1;+ if(!fseek(f,0,SEEK_END))size=ftell(f);+ if(size<0||size>(16L<<20)||fseek(f,0,SEEK_SET)||!(text=(char*)malloc((size_t)size+1))||+ fread(text,1,(size_t)size,f)!=(size_t)size){+ fclose(f);q38_int4_refuse(dir,"index.json cannot be read");+ }+ fclose(f);text[size]=0;+ jval *root=json_parse(text,&arena);+ if(!root||root->t!=J_OBJ)q38_int4_refuse(dir,"index.json is not a JSON object");+ const char *format=jstr(root,"format"),*codes=jstr(root,"codes"),*scales=jstr(root,"scales");+ if(!format||strcmp(format,Q38_INT4_FORMAT)||q38_json_int(root,"version")!=Q38_INT4_VERSION||+ !codes||strcmp(codes,"planar64-v8")||!scales||strcmp(scales,"f32-per-64")||+ q38_json_int(root,"group_size")!=XF_BLOCK)+ q38_int4_refuse(dir,"index.json is not format " Q38_INT4_FORMAT " version 1, int4 codes v+8 "+ "in planar blocks of 64 with f32 scales per 64");+ Q38Int4Experts *x=(Q38Int4Experts*)calloc(1,sizeof(*x));+ if(!x){fprintf(stderr,"OOM int4 expert index\n");exit(1);}+ int rows[3]={c->inter,c->inter,c->hidden},cols[3]={c->hidden,c->hidden,c->inter};+ for(int k=0;k<3;k++){+ x->code_bytes[k]=(int64_t)rows[k]*q38_int4_row_bytes(cols[k]);+ x->scale_count[k]=(int64_t)rows[k]*q38_int4_groups(cols[k]);+ x->scale_off+=x->code_bytes[k];+ }+ x->record_bytes=x->scale_off++ (x->scale_count[0]+x->scale_count[1]+x->scale_count[2])*(int64_t)sizeof(float);+ if(q38_json_int(root,"layers")!=c->layers||q38_json_int(root,"experts")!=c->experts||+ q38_json_int(root,"hidden_size")!=c->hidden||+ q38_json_int(root,"moe_intermediate_size")!=c->inter||+ q38_json_int(root,"record_bytes")!=x->record_bytes||x->scale_off%(int64_t)sizeof(float))+ q38_int4_refuse(dir,"index.json was written for another geometry than config.json's");+ jval *complete=json_get(root,"complete"),*done=json_get(root,"done");+ if(!complete||complete->t!=J_BOOL||!complete->boolean){+ int layers_done=done&&done->t==J_ARR?done->len:0;+ if(wanted>0){+ fprintf(stderr,"[qwen38] %s: conversion incomplete (%d of %d layers); "+ "Q38_EXPERT_INT4=1 refuses to fall back to FP8\n",dir,layers_done,c->layers);exit(1);+ }+ fprintf(stderr,"[qwen38] %s: conversion incomplete (%d of %d layers); "+ "the routed experts stay as the snapshot has them\n",dir,layers_done,c->layers);+ json_free(root);free(arena);free(text);free(x);+ return;+ }+ json_free(root);free(arena);free(text);+ st_init(&x->S,dir);+ x->fd=(int*)malloc((size_t)c->layers*sizeof(int));+ x->off=(int64_t*)malloc((size_t)c->layers*(size_t)c->experts*sizeof(int64_t));+ if(!x->fd||!x->off){fprintf(stderr,"OOM int4 expert index\n");exit(1);}+ for(int layer=0;layer<c->layers;layer++){+ x->fd[layer]=-1;+ for(int expert=0;expert<c->experts;expert++)x->off[(int64_t)layer*c->experts+expert]=-1;+ }+ for(int layer=m->range_begin;layer<m->range_end;layer++)+ for(int expert=0;expert<c->experts;expert++){+ int fd=-1;int64_t start=-1;+ if(q38_int4_record_at(m,x,layer,expert,&fd,&start)||+ (x->fd[layer]>=0&&fd!=x->fd[layer])){+ char why[160];+ snprintf(why,sizeof why,"layer %d expert %d is missing or is not one int4-g64 "+ "record of %lld bytes",layer,expert,(long long)x->record_bytes);+ q38_int4_refuse(dir,why);+ }+ x->fd[layer]=fd;x->off[(int64_t)layer*c->experts+expert]=start;+ }+ m->x4=x;+}++/* Point a slot's three matrices into one record (a slab or a mapping). */+static void q38_bind_int4_record(Model *m,Slot *slot,uint8_t *record) {+ Cfg *c=&m->c;const Q38Int4Experts *x=m->x4;+ int rows[3]={c->inter,c->inter,c->hidden},cols[3]={c->hidden,c->hidden,c->inter};+ Q38Weight *dst[3]={&slot->gate,&slot->up,&slot->down};+ uint8_t *codes=record;float *scales=(float*)(record+x->scale_off);+ for(int k=0;k<3;k++){+ Q38Weight *weight=dst[k];q38_weight_free(weight);+ weight->data=codes;weight->scales=scales;weight->rows=rows[k];weight->cols=cols[k];+ weight->elements=(int64_t)rows[k]*cols[k];weight->scale_count=x->scale_count[k];+ weight->kind=Q38_WEIGHT_INT4G64;+ codes+=x->code_bytes[k];scales+=x->scale_count[k];+ }+}++/* One read per miss, into the slot's own slab; with COLI_MAP_EXPERTS=1 the+ * slot points into the mapped file instead, as the FP8 path does. Safe from+ * the parallel batch loader: every job owns a distinct slot. */+static void q38_load_int4_record(Model *m,int layer,int expert,Slot *slot) {+ Q38Int4Experts *x=m->x4;+ int fd=x->fd[layer];int64_t off=x->off[(int64_t)layer*m->c.experts+expert];+ const uint8_t *mapped=(const uint8_t*)st_map_shard_range(fd,off,x->record_bytes);+ if(mapped){+ q38_bind_int4_record(m,slot,(uint8_t*)mapped);+ free(slot->int4_slab);slot->int4_slab=NULL;slot->int4_slab_bytes=0;+ return;+ }+ if(!slot->int4_slab||slot->int4_slab_bytes!=x->record_bytes){+ void *replacement=realloc(slot->int4_slab,(size_t)x->record_bytes);+ if(!replacement){fprintf(stderr,"OOM int4 expert record\n");exit(1);}+ slot->int4_slab=replacement;slot->int4_slab_bytes=x->record_bytes;+ }+ q38_bind_int4_record(m,slot,(uint8_t*)slot->int4_slab);+ st_read_range_raw_cap(&x->S,fd,off,x->record_bytes,slot->int4_slab,+ slot->int4_slab_bytes,1,"pread Qwen3.8 int4 expert");+}+static void q38_load_expert(Model *m,int layer,int eid,Slot *s) {Cfg *c=&m->c; int H=c->hidden,I=c->inter; char nm[320],sn[340];+ if(m->x4){+ double started=now_s();+ q38_load_int4_record(m,layer,eid,s);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;+ return;+ }q38_name(m,nm,sizeof nm,layer,"mlp.experts.gate_up_proj");if(st_has(&m->S,nm)){st_tensor *t=st_find(&m->S,nm);@@ -1490,7 +1757,7 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,if(count>cache->cap)return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY,layer,cache->cap,count);- if(!q38_prepare_expert_scale_bank(m,layer))+ if(!m->x4&&!q38_prepare_expert_scale_bank(m,layer))return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK,layer,0,0);/* The demand set is no longer bounded by the decode top-k: the MoE prefill* hands over the whole chunk union (up to the cache cap) so its loads run@@ -1516,7 +1783,7 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,continue;}st_tensor *weight[3];- if(!q38_native_fp8_expert_tensors(m,layer,expert,weight)){+ if(!m->x4&&!q38_native_fp8_expert_tensors(m,layer,expert,weight)){free(jobs);return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_LAYOUT,layer,expert,0);@@ -1551,7 +1818,7 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,if(slot->eid>=0)cache->by_expert[slot->eid]=-1;}slot->eid=-1;jobs[job_count].expert=expert;jobs[job_count].slot=slot;- if(!q38_native_fp8_expert_tensors(m,layer,expert,jobs[job_count].weight)){+ if(!m->x4&&!q38_native_fp8_expert_tensors(m,layer,expert,jobs[job_count].weight)){fprintf(stderr,"Qwen3.8 expert layout changed during batch reservation\n");exit(1);}job_count++;@@ -1569,12 +1836,14 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,#endifdouble started=now_s();#pragma omp parallel for schedule(static) num_threads(workers) if(job_count>1)- for(int job=0;job<job_count;job++)- q38_load_native_fp8_ranges(m,layer,jobs[job].expert,jobs[job].slot,- jobs[job].weight);+ for(int job=0;job<job_count;job++){+ if(m->x4)q38_load_int4_record(m,layer,jobs[job].expert,jobs[job].slot);+ else q38_load_native_fp8_ranges(m,layer,jobs[job].expert,jobs[job].slot,+ jobs[job].weight);+ }q38_tm_add(m,Q38_TM_EXPERT_READ,started);- m->expert_weight_reads+=(uint64_t)job_count*2;- m->expert_pair_reads+=(uint64_t)job_count;+ m->expert_weight_reads+=(uint64_t)job_count*(m->x4?1:2);+ if(!m->x4)m->expert_pair_reads+=(uint64_t)job_count;if(job_count>1)m->expert_parallel_batches++;for(int job=0;job<job_count;job++){Slot *slot=jobs[job].slot;slot->eid=jobs[job].expert;@@ -2127,6 +2396,13 @@ static void q38_tier_start(Model *m,int cap) {const char *on=getenv("COLI_CUDA");if(!on||on[0]!='1'||on[1])return;Cfg *c=&m->c;+ if(m->x4){+ /* The tier streams e4m3 bytes (fmt 8) and copies them at qt_note; its+ * int4 mode wants every expert resident in RAM. Neither is this. */+ fprintf(stderr,"[qtier] qwen38: the expert tier streams native FP8 experts only; the int4-g64 "+ "experts stay on the CPU and the tier stays off (Q38_EXPERT_INT4=0 gives it the FP8 ones)\n");+ return;+ }if(!m->native_fp8){fprintf(stderr,"[qtier] qwen38: expert tier needs native FP8 experts (Q38_NATIVE_FP8=1); staying on the CPU\n");return;@@ -2163,7 +2439,7 @@ static void q38_moe_decode(Model *m,Layer *l,int layer,const float *x,int S,floauint32_t qmask=qt_issue(layer,idx,K,xs);int cpu_idx[Q38_MAX_TOPK],cpu_rank[Q38_MAX_TOPK],cpu_n=0;for(int z=0;z<K;z++)if(!((qmask>>z)&1u)){cpu_idx[cpu_n]=idx[z];cpu_rank[cpu_n]=z;cpu_n++;}- q38_prefetch_native_fp8_experts(m,layer,cpu_idx,cpu_n);+ q38_prefetch_experts(m,layer,cpu_idx,cpu_n);double phase_started=now_s();q38_weight_matmul(sg,xs,&l->sh_g,1,H,SI);q38_weight_matmul(su,xs,&l->sh_u,1,H,SI);for(int j=0;j<SI;j++)sh[j]=q38_silu(sg[j])*su[j];q38_weight_matmul(shared,sh,&l->sh_d,1,SI,H);@@ -2318,7 +2594,7 @@ static void q38_moe_prefill(Model *m,Layer *l,int layer,const float *x,}/* One advice range per distinct expert is enough for this chunk. */- q38_prefetch_native_fp8_experts(m,layer,unique,unique_count);+ q38_prefetch_experts(m,layer,unique,unique_count);/* Shared expert work is independent across rows and remains resident;* batching it here also keeps its cost out of the routed groups. */@@ -2610,6 +2886,7 @@ static void q38_model_free(Model *m) {q38_weight_free(&m->cache[i].slots[s].up);q38_weight_free(&m->cache[i].slots[s].down);free(m->cache[i].slots[s].fp8_slab);+ free(m->cache[i].slots[s].int4_slab);}}free(m->cache[i].slots); free(m->cache[i].by_expert);@@ -2618,6 +2895,7 @@ static void q38_model_free(Model *m) {free(m->DN_rec ? m->DN_rec[i] : NULL); free(m->DN_conv ? m->DN_conv[i] : NULL);free(m->K ? m->K[i] : NULL); free(m->V ? m->V[i] : NULL); free(m->IK ? m->IK[i] : NULL);}+ if(m->x4){st_destroy(&m->x4->S);free(m->x4->fd);free(m->x4->off);free(m->x4);}free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);q38_weight_free(&m->embed);q38_weight_free(&m->lm_head);free(m->final_gr.norm);q38_weight_free(&m->final_gr.down);q38_weight_free(&m->final_gr.up);q38_weight_free(&m->final_gr.inject);diff --git a/c/tests/test_qwen38_int4_convert.py b/c/tests/test_qwen38_int4_convert.pynew file mode 100644index 00000000..02f88109--- /dev/null+++ b/c/tests/test_qwen38_int4_convert.py@@ -0,0 +1,402 @@+"""tools/convert_qwen38_experts_int4.py: the int4-g64 sidecar of Qwen3.8's routed experts.++As a unittest (numpy only, part of `make test`) it converts a synthetic FP8+snapshot whose widths have partial FP8 blocks and an int4 tail, then reads the+sidecar back with its own parser and checks, against the source dequantized+with quant.h's own E4M3_LUT (parsed from the C source):+ * the layout the engine validates: data on a 4096-byte boundary, every+ expert one record of six tensors back to back, index.json complete;+ * the quantizer: every scale is its group's absmax / 7 in f32, every value+ within half a scale of the source, relative L2 error that of int4-g64;+ * resume: a second run rewrites nothing, a removed layer is converted again+ to the same bytes, a changed snapshot is refused.++As a script it verifies an existing sidecar against its snapshot and prints the+relative L2 error of every expert matrix (sampled with --layers/--experts on+the real model), and with --engine it checks that qwen38 refuses a sidecar that+disagrees with its index and falls back from an incomplete one:++ python3 tests/test_qwen38_int4_convert.py --model ./qwen38_tiny_int4 --engine ./qwen38+"""+import argparse+import json+import math+import os+import shutil+import struct+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path++try:+ import numpy as np+ HAVE_NUMPY = True+except ImportError:+ HAVE_NUMPY = False++HERE = Path(__file__).resolve().parent+TOOLS = HERE.parent / "tools"+QUANT_H = HERE.parent / "quant.h"+PROJ = ("gate_proj", "up_proj", "down_proj")+++# ---------- independent readers ----------++def engine_e4m3_table():+ """quant.h's E4M3_LUT, read from the C source the engine compiles."""+ text = QUANT_H.read_text(encoding="utf-8")+ body = text[text.index("E4M3_LUT[256] = {") + len("E4M3_LUT[256] = {"):]+ body = body[:body.index("};")]+ values = []+ for item in body.replace("\n", " ").split(","):+ item = item.strip().rstrip("f")+ if not item:+ continue+ values.append(math.nan if item.upper() == "NAN" else float.fromhex(item))+ assert len(values) == 256, len(values)+ return np.array(values, np.float32)+++def read_safetensors(path):+ """{name: (dtype, shape, absolute offset, nbytes)} and the data start."""+ with open(path, "rb") as f:+ n = struct.unpack("<Q", f.read(8))[0]+ header = json.loads(f.read(n))+ tensors = {k: (v["dtype"], v["shape"], 8 + n + v["data_offsets"][0],+ v["data_offsets"][1] - v["data_offsets"][0])+ for k, v in header.items() if k != "__metadata__"}+ return tensors, 8 + n, header.get("__metadata__", {})+++def read_bytes(path, offset, nbytes):+ with open(path, "rb") as f:+ f.seek(offset)+ return f.read(nbytes)+++def floats(raw, dtype):+ if dtype == "BF16":+ return (np.frombuffer(raw, np.uint16).astype(np.uint32) << 16).view(np.float32)+ if dtype == "F16":+ return np.frombuffer(raw, np.float16).astype(np.float32)+ return np.frombuffer(raw, np.float32)+++class Snapshot:+ """The routed experts of a Qwen3.8 snapshot, dequantized as the engine's+ expanded FP8 path does (byte -> E4M3_LUT, times the f32 block scale)."""++ def __init__(self, model_dir):+ self.dir = Path(model_dir)+ config = json.loads((self.dir / "config.json").read_text(encoding="utf-8"))+ text = config.get("text_config", config)+ self.layers, self.experts = text["num_hidden_layers"], text["num_experts"]+ self.hidden, self.inter = text["hidden_size"], text["moe_intermediate_size"]+ self.where = {}+ for shard in sorted(self.dir.glob("*.safetensors")):+ tensors, _, _ = read_safetensors(shard)+ for name, info in tensors.items():+ self.where[name] = (shard, info)+ self.prefix = next(p for p in ("model.language_model", "model")+ if p + ".embed_tokens.weight" in self.where)+ self.lut = engine_e4m3_table()++ def matrix(self, layer, expert, k):+ name = f"{self.prefix}.layers.{layer}.mlp.experts.{expert}.{PROJ[k]}.weight"+ shard, (dtype, shape, off, n) = self.where[name]+ rows, cols = shape+ raw = read_bytes(shard, off, n)+ if dtype.startswith("F8") or dtype == "float8_e4m3fn":+ sshard, (sdtype, sshape, soff, sn) = self.where[name + "_scale_inv"]+ block = floats(read_bytes(sshard, soff, sn), sdtype).reshape(sshape)+ out = np.empty((rows, cols), np.float32)+ codes = np.frombuffer(raw, np.uint8).reshape(rows, cols)+ for r in range(rows):+ for c0 in range(0, cols, 128):+ out[r, c0:c0 + 128] = self.lut[codes[r, c0:c0 + 128]] * block[r // 128, c0 // 128]+ return out+ return floats(raw, dtype).reshape(rows, cols)+++def planar_codes(packed_row, cols):+ """One int4-g64 row's codes (0..15), decoded element by element."""+ out = np.empty(cols, np.int32)+ body = cols // 64 * 64+ for i in range(cols):+ if i < body:+ block, k = packed_row[(i // 64) * 32:(i // 64) * 32 + 32], i % 64+ out[i] = block[k] & 15 if k < 32 else block[k - 32] >> 4+ else:+ byte = packed_row[i >> 1]+ out[i] = byte >> 4 if i & 1 else byte & 15+ return out+++class Sidecar:+ def __init__(self, sidecar_dir):+ self.dir = Path(sidecar_dir)+ self.index = json.loads((self.dir / "index.json").read_text(encoding="utf-8"))+ self.files = {}++ def layer(self, layer):+ if layer not in self.files:+ self.files[layer] = read_safetensors(self.dir / f"layer-{layer:03d}.safetensors")+ return self.files[layer]++ def expert(self, prefix, layer, expert, shapes):+ """[(codes [R, C] 0..15, scales [R, G])] for gate, up, down, after checking+ that the six tensors are one record in record order."""+ tensors, data_start, _ = self.layer(layer)+ path = self.dir / f"layer-{layer:03d}.safetensors"+ base = f"{prefix}.layers.{layer}.mlp.experts.{expert}"+ names = [f"{base}.{p}.weight" for p in PROJ] + [f"{base}.{p}.weight.qs" for p in PROJ]+ start, at, parts = tensors[names[0]][2], 0, []+ for i, name in enumerate(names):+ dtype, shape, off, n = tensors[name]+ rows, cols = shapes[i % 3]+ want = ("U8", [rows, (cols + 1) // 2]) if i < 3 else ("F32", [rows, (cols + 63) // 64])+ if (dtype, shape) != want or off != start + at:+ raise AssertionError(f"{name}: {dtype} {shape} at +{off - start}, "+ f"expected {want[0]} {want[1]} at +{at}")+ parts.append(read_bytes(path, off, n))+ at += n+ if at != self.index["record_bytes"] or start % 4:+ raise AssertionError(f"layer {layer} expert {expert}: record of {at} bytes at {start}")+ out = []+ for k in range(3):+ rows, cols = shapes[k]+ packed = np.frombuffer(parts[k], np.uint8).reshape(rows, (cols + 1) // 2)+ codes = np.stack([planar_codes(packed[r], cols) for r in range(rows)])+ scales = np.frombuffer(parts[3 + k], np.float32).reshape(rows, (cols + 63) // 64)+ out.append((codes, scales))+ return out+++def verify(model_dir, layers=None, experts=None, quiet=False):+ """Every sampled expert matrix: the quantizer's invariants hold and the+ relative L2 error against the source. Returns the list of errors."""+ snap = Snapshot(model_dir)+ side = Sidecar(snap.dir / "experts-int4g64")+ idx = side.index+ assert idx["format"] == "colibri.qwen38.experts-int4g64" and idx["version"] == 1, idx["format"]+ assert idx["group_size"] == 64 and idx["codes"] == "planar64-v8" and idx["scales"] == "f32-per-64"+ assert (idx["layers"], idx["experts"], idx["hidden_size"], idx["moe_intermediate_size"]) == \+ (snap.layers, snap.experts, snap.hidden, snap.inter)+ assert idx["complete"] is True and idx["done"] == list(range(snap.layers)), "incomplete sidecar"+ shapes = ((snap.inter, snap.hidden), (snap.inter, snap.hidden), (snap.hidden, snap.inter))+ errors = []+ for layer in layers if layers is not None else range(snap.layers):+ _, data_start, meta = side.layer(layer)+ assert data_start % 4096 == 0, f"layer {layer}: data starts at {data_start}"+ assert meta.get("layer") == str(layer), meta+ for expert in experts if experts is not None else range(snap.experts):+ stored = side.expert(snap.prefix, layer, expert, shapes)+ for k, (codes, scales) in enumerate(stored):+ w = snap.matrix(layer, expert, k)+ rows, cols = w.shape+ g = (cols + 63) // 64+ padded = np.zeros((rows, g * 64), np.float32)+ padded[:, :cols] = w+ amax = np.abs(padded.reshape(rows, g, 64)).max(axis=2)+ expect = np.maximum(amax, np.float32(1e-12)) / np.float32(7.0)+ if not np.array_equal(scales, expect):+ raise AssertionError(f"layer {layer} expert {expert} {PROJ[k]}: scales are not absmax/7")+ values = (codes.astype(np.float32) - 8.0) * np.repeat(scales, 64, axis=1)[:, :cols]+ slack = np.repeat(scales, 64, axis=1)[:, :cols] * np.float32(0.5 + 1e-5)+ if not (np.abs(values - w) <= slack).all():+ raise AssertionError(f"layer {layer} expert {expert} {PROJ[k]}: a value is "+ f"further than half a scale from its source")+ err = float(np.linalg.norm((values - w).astype(np.float64)) /+ max(np.linalg.norm(w.astype(np.float64)), 1e-30))+ errors.append(err)+ if not quiet:+ print(f" layer {layer:3d} expert {expert:3d} {PROJ[k]:9s} rel L2 {err:.4f}")+ return errors+++# ---------- a synthetic snapshot ----------++def write_safetensors(path, tensors):+ header, blobs, offset = {}, [], 0+ for name, (dtype, shape, raw) in tensors.items():+ header[name] = {"dtype": dtype, "shape": list(shape), "data_offsets": [offset, offset + len(raw)]}+ blobs.append(raw)+ offset += len(raw)+ encoded = json.dumps(header).encode()+ encoded += b" " * (-len(encoded) % 8)+ with open(path, "wb") as f:+ f.write(struct.pack("<Q", len(encoded)))+ f.write(encoded)+ for blob in blobs:+ f.write(blob)+++def make_snapshot(root, layers=2, experts=3, hidden=192, inter=72, seed=7):+ """Per-expert FP8 experts with BF16 scale_inv, gate/up and down in two shards+ plus the index, as the release ships them. hidden 192 is one and a half FP8+ blocks and three int4 groups; inter 72 is one group plus a tail of 8."""+ rng = np.random.default_rng(seed)+ prefix = "model.language_model"+ shards = {"model-00001-of-00002.safetensors": {}, "model-00002-of-00002.safetensors": {}}+ shards["model-00001-of-00002.safetensors"][f"{prefix}.embed_tokens.weight"] = \+ ("F32", [4, hidden], np.zeros((4, hidden), np.float32).tobytes())+ for layer in range(layers):+ for expert in range(experts):+ for k, (rows, cols) in enumerate(((inter, hidden), (inter, hidden), (hidden, inter))):+ codes = rng.integers(0, 256, size=(rows, cols), dtype=np.uint8)+ codes[(codes & 0x7F) == 0x7F] = 0x3C # no NaN encodings+ scale = (rng.random((-(-rows // 128), -(-cols // 128))) * 0.02 + 0.001).astype(np.float32)+ bf16 = (scale.view(np.uint32) >> 16).astype(np.uint16)+ name = f"{prefix}.layers.{layer}.mlp.experts.{expert}.{PROJ[k]}.weight"+ shard = "model-00002-of-00002.safetensors" if k == 2 else "model-00001-of-00002.safetensors"+ shards[shard][name] = ("F8_E4M3", [rows, cols], codes.tobytes())+ shards[shard][name + "_scale_inv"] = ("BF16", list(scale.shape), bf16.tobytes())+ weight_map = {}+ for fname, tensors in shards.items():+ write_safetensors(root / fname, tensors)+ weight_map.update({name: fname for name in tensors})+ (root / "model.safetensors.index.json").write_text(json.dumps({"weight_map": weight_map}))+ (root / "config.json").write_text(json.dumps({"text_config": {+ "model_type": "qwen4_exp_text", "num_hidden_layers": layers, "num_experts": experts,+ "hidden_size": hidden, "moe_intermediate_size": inter}}))+++def import_converter():+ sys.path.insert(0, str(TOOLS))+ import convert_qwen38_experts_int4 as converter+ return converter++[email protected](HAVE_NUMPY, "numpy is required")+class ConvertQwen38ExpertsInt4(unittest.TestCase):+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.root = Path(self.tmp.name)+ make_snapshot(self.root)+ self.converter = import_converter()++ def tearDown(self):+ self.tmp.cleanup()++ def convert(self, **kw):+ from contextlib import redirect_stdout+ import io+ buf = io.StringIO()+ with redirect_stdout(buf):+ self.converter.convert(str(self.root), workers=1, **kw)+ return buf.getvalue()++ def test_layout_quantizer_and_error(self):+ self.convert()+ index = json.loads((self.root / "experts-int4g64" / "index.json").read_text())+ self.assertTrue(index["complete"])+ self.assertEqual(index["record_bytes"], 2 * 72 * 96 + 192 * 36 + 4 * (2 * 72 * 3 + 192 * 2))+ errors = verify(self.root, quiet=True)+ self.assertEqual(len(errors), 2 * 3 * 3)+ mean = sum(errors) / len(errors)+ self.assertTrue(0.05 < mean < 0.15 and max(errors) < 0.2, (mean, max(errors)))++ def test_matches_the_engine_table(self):+ self.assertTrue(np.array_equal(self.converter.E4M3, engine_e4m3_table(), equal_nan=True))++ def test_resume(self):+ side = self.root / "experts-int4g64"+ self.convert(layers="0")+ index = json.loads((side / "index.json").read_text())+ self.assertEqual((index["done"], index["complete"]), ([0], False))+ first = (side / "layer-000.safetensors").read_bytes()+ mtime = (side / "layer-000.safetensors").stat().st_mtime_ns+ out = self.convert()+ self.assertIn("1 layer(s) already converted", out)+ self.assertEqual((side / "layer-000.safetensors").stat().st_mtime_ns, mtime)+ self.assertTrue(json.loads((side / "index.json").read_text())["complete"])+ # a layer whose file is gone is converted again, to the same bytes+ (side / "layer-000.safetensors").unlink()+ out = self.convert()+ self.assertIn("converting it again", out)+ self.assertEqual((side / "layer-000.safetensors").read_bytes(), first)+ # a sidecar started from another snapshot is not resumed+ config = json.loads((self.root / "config.json").read_text())+ config["note"] = "changed"+ (self.root / "config.json").write_text(json.dumps(config))+ with self.assertRaises(SystemExit):+ self.convert()+++# ---------- engine checks on a converted fixture ----------++def engine_refusals(model_dir, engine):+ """qwen38 against edited copies of a converted snapshot."""+ def run(snap, **env):+ full = dict(os.environ, OMP_NUM_THREADS="2", SNAP=str(snap), **env)+ p = subprocess.run([engine, "2", "8", str(snap / "ref.json")], env=full,+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)+ return p.returncode, p.stderr++ def edited(edit_index=None, truncate=False):+ tmp = Path(tempfile.mkdtemp())+ snap = tmp / "snap"+ shutil.copytree(model_dir, snap)+ index_path = snap / "experts-int4g64" / "index.json"+ if edit_index:+ index = json.loads(index_path.read_text())+ edit_index(index)+ index_path.write_text(json.dumps(index))+ if truncate:+ layer = snap / "experts-int4g64" / "layer-001.safetensors"+ data = layer.read_bytes()+ with open(layer, "wb") as f:+ header = struct.unpack("<Q", data[:8])[0]+ f.write(data[:8 + header + 64])+ return tmp, snap++ failures = 0+ cases = [+ ("an index for another geometry is refused", dict(edit_index=lambda i: i.update(hidden_size=i["hidden_size"] * 2)), {}, "refusing", True),+ ("an index of another format version is refused", dict(edit_index=lambda i: i.update(version=2)), {}, "refusing", True),+ ("a layer file shorter than its header says is refused", dict(truncate=True), {}, "out of file bounds", True),+ ("an incomplete sidecar falls back to the snapshot's experts", dict(edit_index=lambda i: i.update(complete=False)), {}, "conversion incomplete", False),+ ("Q38_EXPERT_INT4=1 refuses an incomplete sidecar", dict(edit_index=lambda i: i.update(complete=False)), {"Q38_EXPERT_INT4": "1"}, "refuses to fall back", True),+ ]+ for what, edit, env, needle, must_fail in cases:+ tmp, snap = edited(**edit)+ try:+ rc, err = run(snap, **env)+ finally:+ shutil.rmtree(tmp, ignore_errors=True)+ ok = needle in err and ((rc != 0) if must_fail else (rc == 0 and "native FP8" in err))+ print(f" {'ok ' if ok else 'FAIL'} {what}")+ failures += not ok+ return failures+++def main():+ ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ ap.add_argument("--model", required=True, help="a snapshot with experts-int4g64/")+ ap.add_argument("--layers", help="comma list of layers to check (default: all)")+ ap.add_argument("--experts", help="comma list of experts to check (default: all)")+ ap.add_argument("--engine", help="qwen38 binary: also check its refusals on edited copies")+ a = ap.parse_args()+ layers = [int(x) for x in a.layers.split(",")] if a.layers else None+ experts = [int(x) for x in a.experts.split(",")] if a.experts else None+ print(f"{a.model}/experts-int4g64 against the snapshot's experts:")+ errors = verify(a.model, layers, experts)+ mean = sum(errors) / len(errors)+ print(f" {len(errors)} matrices: rel L2 mean {mean:.4f}, max {max(errors):.4f} "+ f"(every scale absmax/7, every value within half a scale)")+ failures = 0 if 0.02 < mean < 0.2 and max(errors) < 0.25 else 1+ if failures:+ print(" FAIL relative L2 error outside what int4-g64 gives (mean 2-20%, max 25%)")+ if a.engine:+ print("qwen38 on edited copies:")+ failures += engine_refusals(Path(a.model), a.engine)+ print("test_qwen38_int4_convert: " + ("ok" if not failures else f"{failures} failure(s)"))+ return 1 if failures else 0+++if __name__ == "__main__":+ if len(sys.argv) > 1:+ sys.exit(main())+ unittest.main()diff --git a/c/tests/test_qwen38_int4_experts.c b/c/tests/test_qwen38_int4_experts.cnew file mode 100644index 00000000..e2af9bc0--- /dev/null+++ b/c/tests/test_qwen38_int4_experts.c@@ -0,0 +1,209 @@+/* qwen38's routed experts as int4-g64 (the experts-int4g64/ sidecar written by+ * tools/convert_qwen38_experts_int4.py).+ *+ * Pinned with no model:+ * - q38_weight_matmul on an int4-g64 weight equals the dequantized reference+ * ((code - 8) times its group's scale, summed in double) to float rounding,+ * on the release's two expert shapes and on widths whose tail is stored in+ * pairs, for one row (decode) and for a batch (prefill);+ * - every row of a batch is the same bits as that row alone, so decode and+ * the prefill batch cannot drift apart, and a row of whole blocks is the+ * same bits as expert_ffn.h's scalar reference, its kernel's contract.+ * With a converted snapshot on argv (make qwen38-tiny-int4-check):+ * - the sidecar attaches, and every expert loaded through the single LRU path+ * and through the parallel batch path holds exactly the bytes the sidecar+ * stores under that expert's tensor names, read by name;+ * - its matmuls equal the reference built from those bytes;+ * - the prefetch advises one range per expert the cache does not hold;+ * - the int4 values are this snapshot's experts: against the engine's own+ * expanded FP8 their relative L2 error is what int4-g64 costs (2-20%). */+#define _GNU_SOURCE+#define QWEN38_NO_MAIN+#include "../qwen38.c"+#include "../compat.h" /* setenv/unsetenv: MinGW has neither */++static int fails;+static void ck(int ok,const char *what){ if(ok){printf(" ok %s\n",what);return;} printf(" FAIL %s\n",what); fails++; }++static unsigned g_seed=20261002;+static unsigned rnd_u(void){ g_seed=g_seed*1103515245u+12345u; return g_seed>>8; }+static float rnd(void){ return (float)(rnd_u()&0xFFFF)/32768.f-1.f; }++/* element i of an int4-g64 row, decoded without the engine's helpers */+static int code_at(const uint8_t *row,int cols,int i){+ int body=cols/64*64;+ if(i<body){ const uint8_t *blk=row+(i/64)*32; int k=i%64; return k<32?(blk[k]&15):(blk[k-32]>>4); }+ return (i&1)?(row[i>>1]>>4):(row[i>>1]&15);+}++/* y = W x in double, and the sum of |terms| the float error is measured against */+static void reference(double *y,double *mag,const uint8_t *codes,const float *scales,+ const float *x,int S,int I,int O){+ int64_t rb=(I+1)/2,ng=(I+63)/64;+ for(int s=0;s<S;s++)for(int o=0;o<O;o++){+ double a=0,m=0;+ for(int i=0;i<I;i++){+ double t=(double)(code_at(codes+o*rb,I,i)-8)*(double)scales[o*ng+i/64]*(double)x[(int64_t)s*I+i];+ a+=t;m+=fabs(t);+ }+ y[(int64_t)s*O+o]=a;mag[(int64_t)s*O+o]=m;+ }+}++static int within(const float *y,const double *ref,const double *mag,int n,int I){+ double tol=4.0*(sqrt((double)I)+8.0)*FLT_EPSILON;+ for(int k=0;k<n;k++)+ if(!(fabs((double)y[k]-ref[k])<=tol*mag[k]+1e-30))return 0;+ return 1;+}++static void kernel_case(int O,int I,int S){+ int64_t rb=(I+1)/2,ng=(I+63)/64;+ uint8_t *codes=malloc((size_t)(O*rb));float *scales=malloc((size_t)(O*ng)*sizeof(float));+ float *x=malloc((size_t)S*I*sizeof(float)),*y=malloc((size_t)S*O*sizeof(float)),*one=malloc((size_t)O*sizeof(float));+ double *ref=malloc((size_t)S*O*sizeof(double)),*mag=malloc((size_t)S*O*sizeof(double));+ for(int64_t k=0;k<O*rb;k++)codes[k]=(uint8_t)rnd_u(); /* every nibble, the odd tail's pad too */+ for(int64_t k=0;k<O*ng;k++)scales[k]=0.001f+0.05f*(rnd()+1.f);+ for(int64_t k=0;k<(int64_t)S*I;k++)x[k]=rnd()*3.f;+ Q38Weight w={0};+ w.data=codes;w.scales=scales;w.rows=O;w.cols=I;w.elements=(int64_t)O*I;+ w.scale_count=O*ng;w.kind=Q38_WEIGHT_INT4G64;+ q38_weight_matmul(y,x,&w,S,I,O);+ reference(ref,mag,codes,scales,x,S,I,O);+ char what[160];+ snprintf(what,sizeof what,"[%d x %d], %d row(s): equals the dequantized reference to float rounding",O,I,S);+ ck(within(y,ref,mag,S*O,I),what);+ int same=1;+ for(int s=0;s<S&&same;s++){+ q38_weight_matmul(one,x+(int64_t)s*I,&w,1,I,O);+ same=!memcmp(one,y+(int64_t)s*O,(size_t)O*sizeof(float));+ }+ snprintf(what,sizeof what,"[%d x %d]: each batch row is the bits of that row alone",O,I);+ if(S>1)ck(same,what);+ if(I%64==0){+ int exact=1;+ for(int o=0;o<O&&exact;o++)+ exact=y[o]==xf_dot_f32_ref(codes+o*rb,scales+o*ng,x,I);+ snprintf(what,sizeof what,"[%d x %d]: bit-identical to expert_ffn.h's scalar reference",O,I);+ ck(exact,what);+ }+ free(codes);free(scales);free(x);free(y);free(one);free(ref);free(mag);+}++/* the bytes the sidecar stores under this expert's names, read by name */+static uint8_t *stored(Model *m,int layer,int expert,int k,int scale,int64_t *bytes){+ const char *projection[3]={"gate_proj","up_proj","down_proj"};+ char suffix[192],name[320];+ snprintf(suffix,sizeof suffix,"mlp.experts.%d.%s.weight%s",expert,projection[k],scale?".qs":"");+ q38_name(m,name,sizeof name,layer,suffix);+ st_tensor *t=st_find(&m->x4->S,name);+ if(!t)return NULL;+ uint8_t *out=malloc((size_t)t->nbytes);+ st_read_raw(&m->x4->S,name,out,0);*bytes=t->nbytes;+ return out;+}++/* a loaded slot holds the stored bytes, and multiplies like them */+static int slot_matches(Model *m,const Slot *slot,int layer,int expert){+ const Q38Weight *w[3]={&slot->gate,&slot->up,&slot->down};+ int ok=slot->eid==expert;+ for(int k=0;k<3&&ok;k++){+ int64_t nc=0,ns=0;+ uint8_t *codes=stored(m,layer,expert,k,0,&nc),*scales=stored(m,layer,expert,k,1,&ns);+ int O=w[k]->rows,I=w[k]->cols,S=3;+ ok=codes&&scales&&w[k]->kind==Q38_WEIGHT_INT4G64&&+ O==(k<2?m->c.inter:m->c.hidden)&&I==(k<2?m->c.hidden:m->c.inter)&&+ nc==O*q38_int4_row_bytes(I)&&ns==(int64_t)O*q38_int4_groups(I)*(int64_t)sizeof(float)&&+ !memcmp(w[k]->data,codes,(size_t)nc)&&!memcmp(w[k]->scales,scales,(size_t)ns);+ if(ok){+ float *x=malloc((size_t)S*I*sizeof(float)),*y=malloc((size_t)S*O*sizeof(float));+ double *ref=malloc((size_t)S*O*sizeof(double)),*mag=malloc((size_t)S*O*sizeof(double));+ for(int64_t q=0;q<(int64_t)S*I;q++)x[q]=rnd();+ q38_weight_matmul(y,x,w[k],S,I,O);+ reference(ref,mag,codes,(const float*)scales,x,S,I,O);+ ok=within(y,ref,mag,S*O,I);+ free(x);free(y);free(ref);free(mag);+ }+ free(codes);free(scales);+ }+ return ok;+}++static double dequant_gap(Model *m,const Slot *fp8,int layer,int expert,int k){+ const Q38Weight *w=k==0?&fp8->gate:k==1?&fp8->up:&fp8->down;+ int64_t nc=0,ns=0;+ uint8_t *codes=stored(m,layer,expert,k,0,&nc);float *scales=(float*)stored(m,layer,expert,k,1,&ns);+ int O=w->rows,I=w->cols;int64_t rb=(I+1)/2,ng=(I+63)/64;+ double num=0,den=0;+ for(int o=0;o<O;o++)for(int i=0;i<I;i++){+ double source=((const float*)w->data)[(int64_t)o*I+i];+ double v=(double)(code_at(codes+o*rb,I,i)-8)*scales[o*ng+i/64];+ num+=(v-source)*(v-source);den+=source*source;+ }+ free(codes);free(scales);+ return den>0?sqrt(num/den):0;+}++static void fixture_checks(const char *snap){+ printf("the sidecar of %s\n",snap);+ setenv("Q38_EXPERT_INT4","1",1); /* refuse to fall back to FP8 */+ Model m;model_init(&m,snap,1,8);q38_expert_int4_attach(&m,snap);+ ck(m.x4!=NULL,"the sidecar attaches");+ if(!m.x4){q38_model_free(&m);return;}+ int L=m.c.layers,E=m.c.experts,ok=1;+ for(int layer=0;layer<L;layer++)for(int e=0;e<E;e++)+ ok&=slot_matches(&m,q38_expert_get(&m,layer,e),layer,e);+ ck(ok,"single LRU path (cap 1, every expert reloaded): slot bytes are the stored bytes, matmuls equal their reference");+ ck(m.expert_weight_reads==(uint64_t)L*E&&m.miss==m.expert_weight_reads,"one read per miss");+ q38_model_free(&m);++ Model b;model_init(&b,snap,E,8);q38_expert_int4_attach(&b,snap);+ int *all=malloc((size_t)E*sizeof(int));+ Slot **sel=malloc((size_t)E*sizeof(Slot*));+ for(int e=0;e<E;e++)all[e]=E-1-e; /* not in id order */+ q38_prefetch_experts(&b,0,all,E);+ ck(b.expert_prefetch_ranges==(uint64_t)E,"prefetch advises one range per expert not in the cache");+ ok=1;int batched=1;+ for(int layer=0;layer<L;layer++){+ batched&=q38_expert_get_batch(&b,layer,all,E,sel);+ for(int e=0;e<E&&batched;e++)ok&=slot_matches(&b,sel[e],layer,all[e]);+ }+ ck(batched&&b.expert_parallel_batches==(uint64_t)L,"parallel batch path taken for every layer");+ ck(ok,"batch path: slot bytes are the stored bytes, matmuls equal their reference");+ uint64_t prefetched=b.expert_prefetch_ranges;+ q38_prefetch_experts(&b,0,all,E);+ ck(b.expert_prefetch_ranges==prefetched,"nothing is advised once the experts are cached");++ /* the same experts from the snapshot itself, expanded to f32 by the engine */+ setenv("Q38_EXPERT_INT4","0",1);setenv("Q38_NATIVE_FP8","0",1);setenv("Q38_NATIVE_BF16","0",1);+ Model f;model_init(&f,snap,1,8);q38_expert_int4_attach(&f,snap);+ ck(f.x4==NULL,"Q38_EXPERT_INT4=0 leaves the snapshot's experts");+ double worst=0,sum=0;int n=0,kind_ok=1;+ for(int layer=0;layer<L;layer++)for(int e=0;e<E;e++){+ Slot *slot=q38_expert_get(&f,layer,e);+ kind_ok&=slot->gate.kind==Q38_WEIGHT_F32;+ for(int k=0;k<3;k++){double g=dequant_gap(&b,slot,layer,e,k);sum+=g;n++;if(g>worst)worst=g;}+ }+ printf(" int4-g64 against the engine's expanded experts: relative L2 mean %.4f, worst %.4f over %d matrices\n",+ n?sum/n:0,worst,n);+ ck(kind_ok&&n&&sum/n>0.02&&worst<0.2,"the int4 values are this snapshot's experts, within int4-g64's error");+ unsetenv("Q38_EXPERT_INT4");unsetenv("Q38_NATIVE_FP8");unsetenv("Q38_NATIVE_BF16");+ q38_model_free(&f);free(all);free(sel);q38_model_free(&b);+}++int main(int argc,char **argv){+ printf("the int4-g64 matmul\n");+ kernel_case(640,2560,1); /* gate/up of the release */+ kernel_case(640,2560,5);+ kernel_case(2560,640,1); /* down */+ kernel_case(2560,640,9);+ kernel_case(8,32,3); /* the tiny fixture: one short group */+ kernel_case(32,8,3);+ kernel_case(7,130,4); /* two blocks and a two-element tail */+ kernel_case(5,99,2); /* an odd tail: the last byte's high nibble is padding */+ if(argc>1)fixture_checks(argv[1]);+ else printf("(no snapshot on argv: the sidecar checks run from make qwen38-tiny-int4-check)\n");+ if(fails){printf("test_qwen38_int4_experts: %d failure(s)\n",fails);return 1;}+ printf("test_qwen38_int4_experts: ok\n");+ return 0;+}diff --git a/c/tests/test_qwen38_tier_engine.c b/c/tests/test_qwen38_tier_engine.cindex c439df1e..d6020df4 100644--- a/c/tests/test_qwen38_tier_engine.c+++ b/c/tests/test_qwen38_tier_engine.c@@ -10,9 +10,13 @@* oracle tokens/logits must still match exactly; the hit/miss counters pin* that residency was consulted anyway. Real-GPU parity is a hardware run.*- * Needs ./qwen38_tiny_fp8 (make qwen38-tiny-fp8-generate). Include order as- * in test_qwen36_tier_int8_engine.c: engine first, then the fake backend,- * then the tier source, so the tier's statics (G, qs()) are readable here. */+ * Needs ./qwen38_tiny_fp8 (make qwen38-tiny-fp8-generate). Given a snapshot+ * converted to int4-g64 experts on argv (make qwen38-tiny-int4-check passes+ * ./qwen38_tiny_int4), a third pass checks that the tier declines those+ * experts -- it streams e4m3 bytes only -- and that the engine then computes+ * them on the CPU and reproduces the int4 reference. Include order as in+ * test_qwen36_tier_int8_engine.c: engine first, then the fake backend, then+ * the tier source, so the tier's statics (G, qs()) are readable here. */#define main qwen38_main_unused#include "../qwen38.c"#undef main@@ -34,7 +38,7 @@ static void tk(int ok, const char *what) {t_fails++;}-int main(void) {+int main(int argc, char **argv_in) {/* the fixture: 4 layers x 4 experts, hidden 32, inter 8, top-2, all* routed experts native e4m3 with one 128x128 block scale per matrix */setenv("SNAP", "./qwen38_tiny_fp8", 1);@@ -113,6 +117,26 @@ int main(void) {qt_shutdown();tk(qt_dense_count() == 0, "dense handles released at shutdown");+ if (argc > 1) {+ /* Third pass: the int4-g64 sidecar. The tier must stay off -- no+ * upload, no routed expert ever consulted -- and the CPU must answer+ * every expert from the sidecar, matching the int4 reference. */+ printf(" int4-g64 experts (%s)\n", argv_in[1]);+ char ref[2048]; snprintf(ref, sizeof ref, "%s/ref_int4.json", argv_in[1]);+ char *argv4[] = { (char *)"qwen38", (char *)"2", (char *)"8", ref };+ setenv("SNAP", argv_in[1], 1);+ setenv("Q38_EXPERT_INT4", "1", 1);+ fake_uploads = 0; fake_dense_compute = 0;+ unsigned long long before = G.miss; for (int i = 0; i < QT_MAX_DEV; i++) before += G.hits[i];+ rc = qwen38_main_unused(4, argv4);+ tk(rc == 0, "engine with int4 experts under COLI_CUDA=1 reproduces the int4 reference");+ tk(!G.on && !G_fp8_stream, "the tier declined: it streams native FP8 experts only");+ tk(fake_uploads == 0 && qt_dense_count() == 0, "nothing was uploaded, experts or trunk");+ unsigned long long consulted = G.miss; for (int i = 0; i < QT_MAX_DEV; i++) consulted += G.hits[i];+ tk(consulted == before, "qt_issue never took a routed expert off the CPU");+ unsetenv("Q38_EXPERT_INT4");+ }+if (t_fails) { printf("test_qwen38_tier_engine: %d failure(s)\n", t_fails); return 1; }printf("test_qwen38_tier_engine: ok\n");return 0;diff --git a/c/tools/convert_qwen38_experts_int4.py b/c/tools/convert_qwen38_experts_int4.pynew file mode 100644index 00000000..ac65901a--- /dev/null+++ b/c/tools/convert_qwen38_experts_int4.py@@ -0,0 +1,551 @@+#!/usr/bin/env python3+"""Qwen3.8: the routed experts as int4-g64, in a sidecar next to the release.++The release stores every routed expert as FP8 e4m3 with one scale per 128x128+block, 4.9 MB per expert on Qwen3.8-Flash-Next. Decode waits on those bytes:+an expert that is not in the RAM cache is read from disk, and how many fit in+the cache decides how often that happens. This converter writes the same+experts as int4 with one f32 scale per 64 inputs, 2.76 MB per expert, into+<model>/experts-int4g64/. The FP8 shards are only read: the snapshot stays+usable as it is, and qwen38 picks the sidecar up when it is there+(Q38_EXPERT_INT4=0 ignores it).++Only the routed experts change. The shared expert, attention, DeltaNet, PLE,+vision and MTP tensors are not read at all.++Numerics, so the engine and every tier read the bytes they already know:+ dequantize e4m3 byte -> quant.h's E4M3_LUT value, times the block's+ weight_scale_inv (BF16 or F32, widened to f32), one f32 multiply:+ exactly the expanded FP8 path in qwen38_core.h. BF16/F16/F32+ experts (the tiny fixture, fused or per expert) are widened.+ quantize tools/convert_qwen36.py's quantize_row_grouped with --ebits 4+ --gs 64: per row, groups of 64 inputs (a short last group is+ zero-padded), scale = max(absmax, 1e-12) / 7 in f32, code =+ round-half-even(w / scale) clamped to [-8, 7].+ codes stored as v+8 in expert_ffn.h's planar layout: a row is blocks+ of 64 elements, 32 bytes each, byte k holding element k in its+ low nibble and element k+32 in its high nibble. A width that is+ not a multiple of 64 keeps its tail in pairs (byte j = element+ 2j low, 2j+1 high), as idot.h's planarize_i4 leaves it. A row is+ ceil(cols/2) bytes.+ scales f32, [rows][ceil(cols/64)].++Layout: one safetensors file per layer, layer-NNN.safetensors, and index.json.+Every expert is one contiguous record, so a cache miss is one read:++ gate codes U8 [I, H/2] | up codes U8 [I, H/2] | down codes U8 [H, I/2] |+ gate scales F32 [I, H/64] | up scales F32 [I, H/64] | down scales F32 [H, I/64]++named like the release's tensors (<prefix>.layers.L.mlp.experts.E.gate_proj.weight+and the same name + ".qs" for its scales, the .qs convention of the other+int4 containers). Records follow each other with no gap; the data region of+a file starts on a 4096-byte boundary, so on the release's geometry (record+2,764,800 bytes = 675 pages) every record is page aligned.++index.json carries the format name and version, the geometry, the group size,+the record size and the layers already written. It is rewritten (atomically)+after every layer and says "complete": true only when all of them are there;+the engine refuses anything else. A layer is written to a .part file and+renamed when it is whole, so an interrupted run resumes at the first layer+the index does not list.++Usage:+ python3 tools/convert_qwen38_experts_int4.py --model /path/to/qwen38-fp8 --plan+ python3 tools/convert_qwen38_experts_int4.py --model /path/to/qwen38-fp8 [--workers N]++Peak RAM does not grow with the model: each worker holds one expert in flight+(about 0.11 GB with numpy), and the writer at most the records of one layer+(1.4 GB) when the disk lags behind the workers. Measured on a 12-thread+i7-1355U with a synthetic snapshot of the release's expert geometry: 22 s of+compute per layer with 12 workers.+"""+import argparse+import hashlib+import json+import math+import os+import struct+import sys+import time++import numpy as np++try:+ sys.stdout.reconfigure(errors="backslashreplace")+except AttributeError:+ pass++SIDECAR_DIR = "experts-int4g64"+FORMAT = "colibri.qwen38.experts-int4g64"+VERSION = 1+GS = 64+CODES = "planar64-v8"+SCALES = "f32-per-64"+DATA_ALIGN = 4096+PROJ = ("gate_proj", "up_proj", "down_proj")+FP8_BLOCK = 128++_FLOAT_DTYPES = {"BF16": 2, "F16": 2, "F32": 4}+_FP8_DTYPES = ("F8_E4M3", "F8_E4M3FN", "float8_e4m3fn")+++def e4m3_table():+ """quant.h's E4M3_LUT (torch.float8_e4m3fn): bias 7, subnormals at+ exponent 0, only 0x7F/0xFF are NaN (no infinities)."""+ table = np.empty(256, np.float32)+ for byte in range(256):+ exponent, mantissa = (byte >> 3) & 0xF, byte & 7+ if exponent == 0xF and mantissa == 7:+ value = math.nan+ elif exponent == 0:+ value = mantissa / 8.0 * 2.0 ** -6+ else:+ value = (1.0 + mantissa / 8.0) * 2.0 ** (exponent - 7)+ table[byte] = -value if byte & 0x80 else value+ return table+++E4M3 = e4m3_table()+++# ---------- int4-g64: quantize, pack, and their inverses ----------++def groups(cols):+ return (cols + GS - 1) // GS+++def row_bytes(cols):+ return (cols + 1) // 2+++def quantize_g64(w):+ """w f32 [R, C] -> (codes u8 [R, C] holding v+8, scales f32 [R, ceil(C/64)]).+ tools/convert_qwen36.py quantize_row_grouped, bits=4, in numpy."""+ rows, cols = w.shape+ ng = groups(cols)+ pad = ng * GS - cols+ padded = np.pad(w, ((0, 0), (0, pad))) if pad else w+ g = padded.reshape(rows, ng, GS)+ amax = np.abs(g).max(axis=2, keepdims=True)+ scales = np.maximum(amax, np.float32(1e-12)) / np.float32(7.0)+ codes = np.clip(np.rint(g / scales), -8, 7).astype(np.int16) + 8+ return codes.reshape(rows, ng * GS)[:, :cols].astype(np.uint8), scales.reshape(rows, ng)+++def pack_planar(codes):+ """u8 codes [R, C] (0..15) -> u8 [R, ceil(C/2)]: planar blocks of 64, tail in pairs."""+ rows, cols = codes.shape+ body = cols // GS * GS+ parts = []+ if body:+ blocks = codes[:, :body].reshape(rows, body // GS, 2, GS // 2)+ parts.append((blocks[:, :, 0, :] | (blocks[:, :, 1, :] << 4)).reshape(rows, body // 2))+ if body < cols:+ tail = codes[:, body:]+ if tail.shape[1] & 1:+ tail = np.concatenate([tail, np.full((rows, 1), 8, np.uint8)], axis=1)+ parts.append(tail[:, 0::2] | (tail[:, 1::2] << 4))+ return np.ascontiguousarray(np.concatenate(parts, axis=1) if len(parts) > 1 else parts[0])+++def unpack_planar(packed, cols):+ """Inverse of pack_planar: u8 [R, ceil(C/2)] -> codes u8 [R, C]."""+ rows = packed.shape[0]+ body = cols // GS * GS+ out = np.empty((rows, cols), np.uint8)+ if body:+ blocks = packed[:, :body // 2].reshape(rows, body // GS, GS // 2)+ out[:, :body] = np.stack([blocks & 15, blocks >> 4], axis=2).reshape(rows, body)+ if body < cols:+ tail = packed[:, body // 2:]+ pairs = np.stack([tail & 15, tail >> 4], axis=2).reshape(rows, -1)+ out[:, body:] = pairs[:, :cols - body]+ return out+++def dequantize(codes, scales):+ """codes u8 [R, C], scales f32 [R, ceil(C/64)] -> f32 [R, C]."""+ cols = codes.shape[1]+ return (codes.astype(np.float32) - 8.0) * np.repeat(scales, GS, axis=1)[:, :cols]+++def rel_l2(reference, approx):+ """||reference - approx|| / ||reference||, in float64. Plain reductions, not+ np.linalg.norm: that is a BLAS call, and every worker's BLAS starting a+ thread per core made a 12-worker layer 2.4x slower (46 s against 19 s)."""+ ref = reference.astype(np.float64)+ norm = float(np.sum(np.square(ref)))+ diff = float(np.sum(np.square(ref - approx.astype(np.float64))))+ return math.sqrt(diff / norm) if norm > 0 else 0.0+++# ---------- the geometry of one record ----------++class Geometry:+ def __init__(self, layers, experts, hidden, inter):+ self.layers, self.experts, self.hidden, self.inter = layers, experts, hidden, inter+ self.shape = ((inter, hidden), (inter, hidden), (hidden, inter))+ self.code_bytes = [r * row_bytes(c) for r, c in self.shape]+ self.scale_count = [r * groups(c) for r, c in self.shape]+ self.scale_off = sum(self.code_bytes)+ self.record_bytes = self.scale_off + 4 * sum(self.scale_count)+ if self.scale_off % 4:+ raise ValueError(f"hidden={hidden}, moe_intermediate_size={inter}: the codes of an "+ f"expert are {self.scale_off} bytes, not a multiple of 4, so its f32 "+ f"scales could not follow them aligned; this geometry is not supported")++ def tensors(self, prefix, layer, expert):+ """(name, dtype, shape, offset in the record, bytes) in record order."""+ base = f"{prefix}.layers.{layer}.mlp.experts.{expert}"+ out, off = [], 0+ for k, (r, c) in enumerate(self.shape):+ out.append((f"{base}.{PROJ[k]}.weight", "U8", [r, row_bytes(c)], off, self.code_bytes[k]))+ off += self.code_bytes[k]+ for k, (r, c) in enumerate(self.shape):+ n = 4 * self.scale_count[k]+ out.append((f"{base}.{PROJ[k]}.weight.qs", "F32", [r, groups(c)], off, n))+ off += n+ assert off == self.record_bytes+ return out++ def fp8_expert_bytes(self):+ return sum(r * c for r, c in self.shape)+++# ---------- the release, read by offset ----------++def read_header(path):+ with open(path, "rb") as f:+ n = struct.unpack("<Q", f.read(8))[0]+ if n > 512 << 20:+ raise ValueError(f"{path}: safetensors header of {n} bytes")+ return json.loads(f.read(n)), 8 + n+++class Source:+ """The routed experts of a Qwen3.8 snapshot, located by name in its shards."""++ def __init__(self, model_dir):+ self.dir = model_dir+ with open(os.path.join(model_dir, "config.json"), "rb") as f:+ self.config_bytes = f.read()+ config = json.loads(self.config_bytes)+ text = config.get("text_config", config)+ if text.get("model_type") != "qwen4_exp_text":+ sys.exit(f"{model_dir}: text model_type {text.get('model_type')!r} is not qwen4_exp_text")+ self.geometry = Geometry(int(text["num_hidden_layers"]), int(text["num_experts"]),+ int(text["hidden_size"]), int(text["moe_intermediate_size"]))+ index_path = os.path.join(model_dir, "model.safetensors.index.json")+ if os.path.exists(index_path):+ with open(index_path, "rb") as f:+ weight_map = json.load(f)["weight_map"]+ files = sorted(set(weight_map.values()))+ else:+ weight_map = None+ files = sorted(f for f in os.listdir(model_dir) if f.endswith(".safetensors"))+ if not files:+ sys.exit(f"{model_dir}: no safetensors shards")+ self.tensors = {}+ for name in files:+ path = os.path.join(model_dir, name)+ header, base = read_header(path)+ for key, meta in header.items():+ if key == "__metadata__" or (weight_map and weight_map.get(key) != name):+ continue+ if ".mlp.experts." not in key:+ continue+ begin, end = meta["data_offsets"]+ self.tensors[key] = (path, base + begin, end - begin, meta["dtype"], meta["shape"])+ if os.path.exists(index_path):+ prefixes = [p for p in ("model.language_model", "model")+ if p + ".embed_tokens.weight" in weight_map]+ else:+ prefixes = []+ for name in files:+ header, _ = read_header(os.path.join(model_dir, name))+ prefixes += [p for p in ("model.language_model", "model")+ if p + ".embed_tokens.weight" in header]+ if not prefixes:+ sys.exit(f"{model_dir}: no Qwen4-Exp text embedding; is this a Qwen3.8 snapshot?")+ self.prefix = prefixes[0]+ self._fds = {}++ def fingerprint(self):+ """What a resumed run must find unchanged: the config and the shards+ that hold routed experts (name and size)."""+ shards = sorted({t[0] for t in self.tensors.values()})+ return {"config_sha256": hashlib.sha256(self.config_bytes).hexdigest(),+ "shards": {os.path.basename(p): os.path.getsize(p) for p in shards}}++ def _read(self, path, offset, nbytes):+ fd = self._fds.get(path)+ if fd is None:+ fd = self._fds[path] = os.open(path, os.O_RDONLY | getattr(os, "O_BINARY", 0))+ out = bytearray()+ while len(out) < nbytes:+ chunk = os.pread(fd, nbytes - len(out), offset + len(out))+ if not chunk:+ raise IOError(f"{path}: short read at {offset + len(out)}")+ out += chunk+ return bytes(out)++ def _float(self, entry, element_offset, count):+ path, offset, nbytes, dtype, _ = entry+ size = _FLOAT_DTYPES[dtype]+ raw = self._read(path, offset + element_offset * size, count * size)+ if dtype == "BF16":+ return (np.frombuffer(raw, np.uint16).astype(np.uint32) << 16).view(np.float32)+ if dtype == "F16":+ return np.frombuffer(raw, np.float16).astype(np.float32)+ return np.frombuffer(raw, np.float32).copy()++ def matrix(self, layer, expert, k):+ """Expert `expert`'s projection k (gate, up, down) as the engine sees it, f32."""+ rows, cols = self.geometry.shape[k]+ name = f"{self.prefix}.layers.{layer}.mlp.experts.{expert}.{PROJ[k]}.weight"+ entry = self.tensors.get(name)+ if entry is not None:+ path, offset, nbytes, dtype, shape = entry+ if list(shape) != [rows, cols]:+ raise ValueError(f"{name}: shape {shape}, expected {[rows, cols]}")+ if dtype in _FP8_DTYPES:+ if nbytes != rows * cols:+ raise ValueError(f"{name}: {nbytes} bytes for an e4m3 [{rows}, {cols}]")+ scale = self.tensors.get(name + "_scale_inv")+ nb_r, nb_c = -(-rows // FP8_BLOCK), -(-cols // FP8_BLOCK)+ if scale is None or list(scale[4]) != [nb_r, nb_c] or scale[3] not in _FLOAT_DTYPES:+ raise ValueError(f"{name}_scale_inv: missing, or not a [{nb_r}, {nb_c}] float tensor")+ block = self._float(scale, 0, nb_r * nb_c).reshape(nb_r, nb_c)+ codes = np.frombuffer(self._read(path, offset, nbytes), np.uint8).reshape(rows, cols)+ expand = np.repeat(np.repeat(block, FP8_BLOCK, axis=0)[:rows], FP8_BLOCK, axis=1)[:, :cols]+ w = E4M3[codes] * expand+ elif dtype in _FLOAT_DTYPES:+ w = self._float(entry, 0, rows * cols).reshape(rows, cols)+ else:+ raise ValueError(f"{name}: unsupported expert dtype {dtype}")+ else:+ fused = "gate_up_proj" if k < 2 else "down_proj"+ fname = f"{self.prefix}.layers.{layer}.mlp.experts.{fused}"+ entry = self.tensors.get(fname)+ g = self.geometry+ want = [g.experts, 2 * g.inter, g.hidden] if k < 2 else [g.experts, g.hidden, g.inter]+ if entry is None:+ raise ValueError(f"layer {layer} expert {expert}: neither {name} nor {fname}")+ if list(entry[4]) != want or entry[3] not in _FLOAT_DTYPES:+ raise ValueError(f"{fname}: expected a float {want}, found {entry[3]} {entry[4]}")+ first = expert * want[1] * want[2] + (rows * cols if k == 1 else 0)+ w = self._float(entry, first, rows * cols).reshape(rows, cols)+ if not np.isfinite(w).all():+ raise ValueError(f"{name}: non-finite weights; refusing to quantize them")+ return w+++# ---------- one expert, one layer ----------++_SOURCE = None+++def _init_worker(model_dir):+ global _SOURCE+ _SOURCE = Source(model_dir)+++def convert_expert(task):+ """(layer, expert) -> (the record's bytes, relative L2 error of gate, up,+ down, this process's peak RSS in MB)."""+ layer, expert = task+ codes, scales, errors = [], [], []+ for k in range(3):+ w = _SOURCE.matrix(layer, expert, k)+ q, s = quantize_g64(w)+ errors.append(rel_l2(w, dequantize(q, s)))+ codes.append(pack_planar(q).tobytes())+ scales.append(s.astype(np.float32).tobytes())+ return b"".join(codes) + b"".join(scales), errors, peak_rss_mb()+++def layer_file(layer):+ return f"layer-{layer:03d}.safetensors"+++def layer_header(geometry, prefix, layer):+ """The layer file's header, padded so the data region starts on DATA_ALIGN."""+ header, meta = {}, {"format": FORMAT, "version": str(VERSION), "layer": str(layer)}+ for expert in range(geometry.experts):+ base = expert * geometry.record_bytes+ for name, dtype, shape, off, n in geometry.tensors(prefix, layer, expert):+ header[name] = {"dtype": dtype, "shape": shape, "data_offsets": [base + off, base + off + n]}+ header["__metadata__"] = meta+ raw = json.dumps(header, separators=(",", ":")).encode()+ raw += b" " * ((-(8 + len(raw))) % DATA_ALIGN)+ return struct.pack("<Q", len(raw)) + raw+++def layer_file_bytes(geometry, prefix, layer):+ return len(layer_header(geometry, prefix, layer)) + geometry.experts * geometry.record_bytes+++def write_layer(src, out_dir, layer, pool):+ g = src.geometry+ final = os.path.join(out_dir, layer_file(layer))+ part = final + ".part"+ tasks = [(layer, e) for e in range(g.experts)]+ errors, worker_mb = [], 0.0+ with open(part, "wb") as f:+ f.write(layer_header(g, src.prefix, layer))+ results = pool.imap(convert_expert, tasks, chunksize=4) if pool else map(convert_expert, tasks)+ for record, err, rss in results:+ if len(record) != g.record_bytes:+ raise RuntimeError(f"layer {layer}: record of {len(record)} bytes, expected {g.record_bytes}")+ f.write(record)+ errors.append(err)+ worker_mb = max(worker_mb, rss)+ f.flush()+ os.fsync(f.fileno())+ os.replace(part, final)+ return np.array(errors), worker_mb+++def write_index(out_dir, index):+ tmp = os.path.join(out_dir, "index.json.part")+ with open(tmp, "w", encoding="utf-8") as f:+ json.dump(index, f, indent=1)+ f.write("\n")+ f.flush()+ os.fsync(f.fileno())+ os.replace(tmp, os.path.join(out_dir, "index.json"))+++def new_index(src):+ g = src.geometry+ return {+ "format": FORMAT, "version": VERSION, "complete": False,+ "group_size": GS, "codes": CODES, "scales": SCALES,+ "layers": g.layers, "experts": g.experts,+ "hidden_size": g.hidden, "moe_intermediate_size": g.inter,+ "prefix": src.prefix, "record_bytes": g.record_bytes,+ "record": ["gate codes", "up codes", "down codes", "gate scales", "up scales", "down scales"],+ "quantizer": "per 64 inputs: scale = max(absmax, 1e-12)/7 (f32), code = rint(w/scale) "+ "clamped to [-8,7], stored v+8 (tools/convert_qwen36.py quantize_row_grouped)",+ "files": [layer_file(i) for i in range(g.layers)],+ "done": [],+ "source": src.fingerprint(),+ }+++def resume_index(out_dir, src):+ """The index of a previous run on the same snapshot, or a fresh one."""+ path = os.path.join(out_dir, "index.json")+ fresh = new_index(src)+ if not os.path.exists(path):+ return fresh+ with open(path, encoding="utf-8") as f:+ index = json.load(f)+ keys = ("format", "version", "group_size", "codes", "scales", "layers", "experts",+ "hidden_size", "moe_intermediate_size", "prefix", "record_bytes", "source")+ stale = [k for k in keys if index.get(k) != fresh[k]]+ if stale:+ sys.exit(f"{path} was written for something else ({', '.join(stale)} differ); "+ f"remove {out_dir} to convert this snapshot from scratch")+ done = []+ for layer in index.get("done", []):+ path = os.path.join(out_dir, layer_file(layer))+ if os.path.exists(path) and os.path.getsize(path) == layer_file_bytes(src.geometry, src.prefix, layer):+ done.append(layer)+ else:+ print(f"layer {layer}: listed as done but {path} is missing or the wrong size; converting it again")+ fresh["done"] = sorted(done)+ fresh["complete"] = len(done) == src.geometry.layers+ return fresh+++def parse_layers(spec, count):+ if not spec:+ return list(range(count))+ out = set()+ for part in spec.split(","):+ a, _, b = part.partition("-")+ lo, hi = int(a), int(b) if b else int(a)+ if not 0 <= lo <= hi < count:+ sys.exit(f"--layers {spec}: {part} is outside 0..{count - 1}")+ out.update(range(lo, hi + 1))+ return sorted(out)+++def peak_rss_mb():+ """This process's peak resident set in MB (0 where getrusage is missing)."""+ try:+ import resource+ except ImportError:+ return 0.0+ peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss+ return peak / (1 << 20) if sys.platform == "darwin" else peak / 1024+++def convert(model_dir, out_dir=None, workers=1, layers=None, plan=False):+ """Write (or finish) model_dir's sidecar; returns its directory."""+ src = Source(model_dir)+ g = src.geometry+ out_dir = out_dir or os.path.join(model_dir, SIDECAR_DIR)+ total = sum(layer_file_bytes(g, src.prefix, i) for i in range(g.layers))+ fp8 = g.fp8_expert_bytes() * g.experts * g.layers+ print(f"{model_dir}: {g.layers} layers x {g.experts} experts, hidden {g.hidden}, "+ f"moe_intermediate {g.inter}, prefix {src.prefix}")+ print(f" per expert: FP8 {g.fp8_expert_bytes():,} bytes -> int4-g64 {g.record_bytes:,} bytes "+ f"({100 * g.record_bytes / g.fp8_expert_bytes():.1f}%)")+ print(f" sidecar {out_dir}: {total:,} bytes ({total / 2**30:.2f} GiB) "+ f"for {fp8:,} bytes of FP8 experts")+ if plan:+ return out_dir++ os.makedirs(out_dir, exist_ok=True)+ index = resume_index(out_dir, src)+ wanted = parse_layers(layers, g.layers)+ todo = [i for i in wanted if i not in index["done"]]+ if len(wanted) > len(todo):+ print(f" {len(wanted) - len(todo)} layer(s) already converted, skipped")+ write_index(out_dir, index)++ pool = None+ if workers > 1 and todo:+ from multiprocessing import get_context+ pool = get_context("spawn").Pool(workers, initializer=_init_worker, initargs=(model_dir,))+ else:+ _init_worker(model_dir)+ worst = 0.0+ try:+ for layer in todo:+ started = time.time()+ errors, worker_mb = write_layer(src, out_dir, layer, pool)+ index["done"] = sorted(index["done"] + [layer])+ index["complete"] = len(index["done"]) == g.layers+ write_index(out_dir, index)+ worst = max(worst, float(errors.max()))+ print(f" layer {layer:3d}: {g.experts} experts in {time.time() - started:6.1f}s "+ f"rel L2 gate {errors[:, 0].mean():.4f} up {errors[:, 1].mean():.4f} "+ f"down {errors[:, 2].mean():.4f} (max {errors.max():.4f}) "+ f"peak RSS {peak_rss_mb():.0f} MB"+ + (f" + {workers} workers x {worker_mb:.0f} MB" if pool else ""), flush=True)+ finally:+ if pool:+ pool.close()+ pool.join()+ state = "complete" if index["complete"] else f"{len(index['done'])}/{g.layers} layers"+ print(f"{out_dir}: {state}" + (f", worst expert matrix rel L2 {worst:.4f}" if todo else ""))+ return out_dir+++def main():+ ap = argparse.ArgumentParser(description="Qwen3.8 routed experts FP8 -> int4-g64 sidecar")+ ap.add_argument("--model", required=True, help="the Qwen3.8 snapshot (config.json + shards)")+ ap.add_argument("--out", help=f"sidecar directory (default: <model>/{SIDECAR_DIR}, where the engine looks)")+ ap.add_argument("--workers", type=int, default=os.cpu_count() or 1,+ help="processes converting experts in parallel (default: every CPU)")+ ap.add_argument("--layers", help="only these layers, e.g. 0-3,7 (the index stays incomplete)")+ ap.add_argument("--plan", action="store_true", help="print sizes and exit without writing")+ a = ap.parse_args()+ convert(a.model, a.out, a.workers, a.layers, a.plan)+++if __name__ == "__main__":+ main()diff --git a/c/tools/make_qwen38_tiny.py b/c/tools/make_qwen38_tiny.pyindex 6b34b9c5..7e53aeac 100644--- a/c/tools/make_qwen38_tiny.py+++ b/c/tools/make_qwen38_tiny.py@@ -20,6 +20,7 @@ import argparseimport hashlibimport jsonimport random+import shutilimport sysfrom pathlib import Path@@ -220,7 +221,45 @@ def _rewrite_shard_fp8(out: Path, packed):print(f"fp8 experts: {len(packed)} matrices rewritten as F8_E4M3 + BF16 weight_scale_inv")-def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_experts=False):+def _int4_experts_in_model(model, out: Path):+ """Write the int4-g64 sidecar of the fixture just saved, with the converter+ the real model goes through (tools/convert_qwen38_experts_int4.py), then read its bytes+ back and put the dequantized experts into the model in place, in float32:+ every int4 value (code times an f32 scale) is exact there, and the other+ BF16 weights widen exactly, so the reference computed afterwards is the+ arithmetic of the sidecar with nothing rounded on the way. In BF16 the+ reference's own rounding would be larger than the gap it has to see."""+ sys.path.insert(0, str(Path(__file__).resolve().parent))+ import convert_qwen38_experts_int4 as int4+ from safetensors.numpy import load_file+ sidecar = Path(int4.convert(str(out), workers=1))+ model.float()+ stored = {}+ for path in sorted(sidecar.glob("*.safetensors")):+ stored.update(load_file(str(path)))+ with torch.no_grad():+ for name, param in model.named_parameters():+ if name.endswith(".mlp.experts.gate_up_proj"):+ base = name[: -len("gate_up_proj")]+ E, twoI, H = param.shape; I = twoI // 2+ parts = (("gate_proj", slice(0, I), H), ("up_proj", slice(I, twoI), H))+ elif name.endswith(".mlp.experts.down_proj"):+ base = name[: -len("down_proj")]+ E, H, I = param.shape+ parts = (("down_proj", slice(0, H), I),)+ else:+ continue+ for e in range(E):+ for kind, rows, cols in parts:+ key = f"{base}{e}.{kind}.weight"+ codes = int4.unpack_planar(stored[key], cols)+ values = int4.dequantize(codes, stored[key + ".qs"])+ param.data[e, rows, :] = torch.from_numpy(values)+ print(f"int4 experts: {sidecar} written, the model now holds its dequantized values")+++def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_experts=False,+ int4_experts=False, expert_gain=1.0):if max_new < 1:raise ValueError("max_new must be at least 1")random.seed(seed)@@ -292,8 +331,25 @@ def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_e# save/load round trip (and follows the production arithmetic path).model = model.to(dtype=torch.bfloat16)model.eval()+ if expert_gain != 1.0:+ # At the default initialization the routed experts barely reach the+ # logits: zeroing them all moves the final logits less than the+ # oracle's tolerance (cosine 0.99994), so no end-to-end check can see+ # their arithmetic. With a gain of 3 (applied before any quantization,+ # so the scaled values simply are the fixture's experts) the int4+ # fixture's FP8 and int4 references part after the third generated+ # token, and the token gate itself tells one representation from the+ # other. At 4 a routing near-tie flips between the C engine and the+ # reference.+ with torch.no_grad():+ for name, param in model.named_parameters():+ if ".mlp.experts." in name:+ param.mul_(expert_gain)packed = _fp8_experts_in_model(model) if fp8_experts else Noneout.mkdir(parents=True, exist_ok=True)+ # A sidecar left by an earlier run belongs to the weights it was converted+ # from, and qwen38 picks it up by itself: regenerating the fixture drops it.+ shutil.rmtree(out / "experts-int4g64", ignore_errors=True)model.save_pretrained(str(out), safe_serialization=True)if packed:_rewrite_shard_fp8(out, packed)@@ -310,6 +366,7 @@ def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_e"transformers_version": transformers.__version__,"text_only": True,"fp8_experts": bool(fp8_experts),+ **({"expert_gain": expert_gain} if expert_gain != 1.0 else {}),"naming": "upstream Qwen4ExpForCausalLM (no model.language_model prefix)","config_summary": {"hidden_size": config.hidden_size,@@ -326,6 +383,15 @@ def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_eref_path = out / "ref.json"ref_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")print(f"Reference written to {ref_path}")+ if int4_experts:+ _int4_experts_in_model(model, out)+ payload["int4_experts"] = True+ payload["reference_dtype"] = "float32"+ payload.update(_reference(model, prompt_ids, max_new))+ if emit_ref:+ ref_path = out / "ref_int4.json"+ ref_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")+ print(f"int4 reference written to {ref_path}")print(f"Tiny Qwen3.8 fixture written to {out}")print(f"prompt_ids={payload['prompt_ids']}")print(f"full_ids={payload['full_ids']}")@@ -342,10 +408,17 @@ def main():help="routed experts as F8_E4M3 with 128x128 block weight_scale_inv ""sidecars (the release layout); the reference uses the same ""quantized values")+ parser.add_argument("--expert-gain", type=float, default=1.0,+ help="scale the routed experts so they visibly move the logits "+ "(the int4 fixture uses 3)")+ parser.add_argument("--int4-experts", action="store_true",+ help="also convert the routed experts to the experts-int4g64/ sidecar "+ "and write ref_int4.json, the reference of the dequantized sidecar")args = parser.parse_args()prompt = [int(x) for x in args.prompt_ids.split(",") if x.strip()] if args.prompt_ids else Nonebuild(args.out, prompt_ids=prompt, max_new=args.max_new, seed=args.seed, emit_ref=not args.no_ref,- fp8_experts=args.fp8_experts)+ fp8_experts=args.fp8_experts, int4_experts=args.int4_experts,+ expert_gain=args.expert_gain)if __name__ == "__main__":