Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen38: routed experts as int4-g64 from an experts-int4g64/ sidecar

JustVuggaa3db0acf43a
diff --git a/.gitignore b/.gitignore
index ff65ffa1..4fbe610a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -158,6 +158,7 @@ c/kimi_k3_tiny/.coli_usage
c/qwen38_vision_tiny/
c/qwen38_mm_tiny/
c/qwen38_tiny_fp8/
+c/qwen38_tiny_int4/
# MiMo tiny fixture: config.json and ref.json are committed, the rest is regenerated
c/mimo_tiny/*.safetensors
c/mimo_tiny/model.safetensors.index.json
diff --git a/c/Makefile b/c/Makefile
index 468f3504..38ca6559 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1333,6 +1333,60 @@ qwen38-tiny-fp8-check: qwen38-tiny-fp8-generate qwen38$(EXE)
qwen38-tier-engine-check: qwen38-tiny-fp8-generate tests/test_qwen38_tier_engine$(EXE)
./tests/test_qwen38_tier_engine$(EXE)
+# The routed experts as int4-g64: tools/convert_qwen38_experts_int4.py writes
+# the experts-int4g64/ sidecar next to the FP8 shards and qwen38 picks it up.
+# The fixture is the FP8 one with its experts scaled by 3 (at the default
+# initialization zeroing every expert moves the logits less than the oracle's
+# tolerance), converted; ref_int4.json is the float32 reference of the
+# dequantized sidecar. Gates:
+# (a) the int4 matmul equals a dequantized f32 reference, and every expert the
+# single and the batch path load holds the sidecar's bytes
+# (tests/test_qwen38_int4_experts);
+# (b) the converter's round trip against the FP8 source, per matrix, and the
+# engine refusing a sidecar its index does not describe
+# (tests/test_qwen38_int4_convert.py);
+# (c) the engine on the sidecar reproduces ref_int4.json through the single,
+# batched, prefill-batch and mapped paths at caps 1, 2 and 4, with the
+# last logits identical to the bit across them; the FP8 experts still
+# reproduce ref.json and not ref_int4.json, so the gate tells them apart;
+# (d) under COLI_CUDA=1 on the fake backend the tier declines the int4
+# experts (it streams FP8 only) and the CPU still reproduces ref_int4.json.
+.PHONY: qwen38-tiny-int4-generate qwen38-tiny-int4-check
+qwen38-tiny-int4-generate:
+ $(PYTHON) tools/make_qwen38_tiny.py --out ./qwen38_tiny_int4 --fp8-experts --int4-experts --expert-gain 3
+
+qwen38-tiny-int4-check: qwen38-tiny-int4-generate qwen38-tiny-fp8-generate qwen38$(EXE) \
+ tests/test_qwen38_int4_experts$(EXE) tests/test_qwen38_tier_engine$(EXE)
+ OMP_NUM_THREADS=2 ./tests/test_qwen38_int4_experts$(EXE) ./qwen38_tiny_int4
+ $(PYTHON) tests/test_qwen38_int4_convert.py --model ./qwen38_tiny_int4 --engine ./qwen38$(EXE)
+ @d=./qwen38_tiny_int4/logits; rm -rf $$d; mkdir -p $$d; n=0; \
+ for trunk in 1024 0; do for cap in 1 2 4; do for mode in single batched prefill prefill-serial mapped; do \
+ case $$mode in \
+ single) e="Q38_PREFILL_BATCH=0 Q38_EXPERT_PARALLEL_READS=0";; \
+ batched) e="Q38_PREFILL_BATCH=0";; \
+ prefill) e="Q38_PREFILL_BATCH=1";; \
+ prefill-serial) e="Q38_PREFILL_BATCH=1 Q38_EXPERT_PARALLEL_READS=0";; \
+ mapped) e="COLI_MAP_EXPERTS=1";; \
+ esac; \
+ env $$e Q38_EXPERT_INT4=1 Q38_TRUNK_MIN_KB=$$trunk DUMP=$$d/t$$trunk-c$$cap-$$mode.f32 OMP_NUM_THREADS=2 \
+ SNAP=./qwen38_tiny_int4 ./qwen38$(EXE) $$cap 8 ./qwen38_tiny_int4/ref_int4.json > $$d/run.log 2>&1 \
+ || { cat $$d/run.log; echo "FAIL int4 experts, trunk=$$trunk cap=$$cap $$mode"; exit 1; }; \
+ n=$$((n+1)); \
+ done; done; \
+ for f in $$d/t$$trunk-*.f32; do cmp -s $$f $$d/t$$trunk-c1-single.f32 \
+ || { echo "FAIL $$f: last logits differ from the single path"; exit 1; }; done; done; \
+ for trunk in 1024 0; do \
+ Q38_EXPERT_INT4=0 Q38_TRUNK_MIN_KB=$$trunk OMP_NUM_THREADS=2 SNAP=./qwen38_tiny_int4 \
+ ./qwen38$(EXE) 2 8 ./qwen38_tiny_int4/ref.json > $$d/run.log 2>&1 \
+ || { cat $$d/run.log; echo "FAIL FP8 experts of the int4 fixture, trunk=$$trunk"; exit 1; }; \
+ if Q38_EXPERT_INT4=0 Q38_TRUNK_MIN_KB=$$trunk OMP_NUM_THREADS=2 SNAP=./qwen38_tiny_int4 \
+ ./qwen38$(EXE) 2 8 ./qwen38_tiny_int4/ref_int4.json > $$d/run.log 2>&1 \
+ || ! grep -q '^Matching tokens' $$d/run.log; then \
+ cat $$d/run.log; echo "FAIL the FP8 experts pass the int4 reference: the gate cannot tell them apart"; exit 1; fi; \
+ done; \
+ echo "qwen38 int4-g64 experts: ref_int4.json reproduced in $$n configurations, last logits identical across paths; FP8 control ok"
+ ./tests/test_qwen38_tier_engine$(EXE) ./qwen38_tiny_int4
+
# Context-size gates: KV layout, growth across requests, and the attention
# capacity. Includes qwen36.c directly, so no model file is needed.
# Same tier objects as the engine: the test includes qwen36.c, so with CUDA=1
@@ -1553,6 +1607,11 @@ tests/test_dsv41_untrusted_load$(EXE): tests/test_dsv41_untrusted_load.c deepsee
tests/test_qwen38_idot$(EXE): tests/test_qwen38_idot.c qwen38.c $(VK_OBJ)
$(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)
+# The int4-g64 expert matmul with no model (part of test-c); given a converted
+# snapshot on argv it checks the sidecar's loaders too (qwen38-tiny-int4-check).
+tests/test_qwen38_int4_experts$(EXE): tests/test_qwen38_int4_experts.c qwen38.c $(VK_OBJ)
+ $(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)
+
tests/test_qwen38_serve_framing$(EXE): tests/test_qwen38_serve_framing.c qwen38.c $(VK_OBJ)
$(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)
diff --git a/c/qwen38.c b/c/qwen38.c
index d7802e8a..e6343e9b 100644
--- a/c/qwen38.c
+++ b/c/qwen38.c
@@ -1838,6 +1838,8 @@ static int q38_reference_mode(const char *path,int serve_mode){
return length>=5&&!strcmp(path+length-5,".json");
}
+static void q38_expert_report(Model *m, int cap); /* below, beside the expert layout accounting */
+
#ifndef QWEN38_TEST_SERVE
int main(int argc, char **argv) {
/* Physical-core team sizing, as colibri/inkling/kimi_k3/olmoe/deepseek-v41
@@ -1953,8 +1955,10 @@ int main(int argc, char **argv) {
}
Model m; model_init(&m, snap, cap, bits);
+ q38_expert_int4_attach(&m, snap); /* <snap>/experts-int4g64/ when present (Q38_EXPERT_INT4) */
q38_tier_start(&m, cap); /* COLI_CUDA=1: hot experts stream to VRAM (qwen36_tier.c) */
q38_trunk_cpu_int8(&m); /* the trunk's int8 rows on the CPU, BF16 released (Q38_TRUNK_CPU_INT8=0 keeps BF16) */
+ q38_expert_report(&m, cap); /* expert format, bytes per expert, what the cache costs */
#ifdef COLI_VULKAN
/* After the trunk is int8: those rows upload at their first matmul. No
* device (or COLI_VULKAN unset) leaves g_vk_ready 0, the CPU path. */
@@ -2248,6 +2252,52 @@ static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,
*numeric_kinds=kinds;return 0;
}
+/* Said once at startup: the representation the routed-expert cache holds,
+ * what one expert costs in it, what the cache costs full at this cap, and
+ * how many experts per layer the RAM still available now (resident weights
+ * loaded, cache empty) would hold at that size -- the number a cap is chosen
+ * against, and the one the int4-g64 sidecar moves. */
+static void q38_expert_report(Model *m, int cap) {
+ Cfg *c = &m->c;
+ int layers = m->range_end - m->range_begin;
+ uint64_t per_capacity = 0, fixed = 0, snapshot_expert = 0;
+ unsigned kinds = 0;
+ if (layers < 1) return;
+ if (!q38_segment_expert_layout(m, (uint32_t)m->range_begin, (uint32_t)m->range_end,
+ &per_capacity, &fixed, &kinds))
+ snapshot_expert = per_capacity / (uint64_t)layers;
+ const char *what = kinds == Q38_EXPERT_FP8_BLOCK ? "native FP8" :
+ kinds == Q38_EXPERT_FP8_EXPANDED ? "FP8 expanded to f32 (Q38_NATIVE_FP8=0)" :
+ kinds == Q38_EXPERT_BF16 ? (m->native_bf16 ? "BF16" : "BF16 expanded to f32") :
+ kinds == Q38_EXPERT_F16 || kinds == Q38_EXPERT_F32 ? "f32" : "mixed";
+ uint64_t per_expert = snapshot_expert;
+ char extra[160] = "";
+ if (m->x4) {
+ per_expert = (uint64_t)m->x4->record_bytes;
+ snprintf(extra, sizeof extra, " (the snapshot's %s: %llu bytes)", what,
+ (unsigned long long)snapshot_expert);
+ what = "int4-g64 from " Q38_INT4_DIR "/"; /* and no FP8 scale bank is built */
+ } else if (fixed) {
+ snprintf(extra, sizeof extra, " + %.1f MiB resident scale bank", fixed / 1048576.0);
+ }
+ if (!per_expert) {
+ fprintf(stderr, "[qwen38] routed experts: %s, size unknown; cache %d/layer\n", what, cap);
+ return;
+ }
+ double full = (double)per_expert * (double)cap * (double)layers;
+ fprintf(stderr, "[qwen38] routed experts: %s, %llu bytes (%.2f MiB) per expert%s; "
+ "cache %d/layer x %d layers = %.2f GiB full",
+ what, (unsigned long long)per_expert, per_expert / 1048576.0, extra,
+ cap, layers, full / 1073741824.0);
+ double available = compat_mem_available_gb() * 1e9;
+ if (available > 0) {
+ double room = available / ((double)per_expert * (double)layers);
+ int holds = room > (double)c->experts ? c->experts : (int)room;
+ fprintf(stderr, "; %.1f GB available now holds %d/layer", available / 1e9, holds);
+ }
+ fprintf(stderr, "\n");
+}
+
static int q38_segment_cache_capacity(uint64_t bytes_per_capacity,
uint64_t fixed_scale_bytes,int experts,
uint64_t memory_limit,int *capacity){
diff --git a/c/qwen38_core.h b/c/qwen38_core.h
index 31a63c05..3026f336 100644
--- a/c/qwen38_core.h
+++ b/c/qwen38_core.h
@@ -10,6 +10,7 @@
#ifndef COLI_QWEN38_CORE_H
#define COLI_QWEN38_CORE_H
#include "kv_prefix.h"
+#include "expert_ffn.h" /* the int4-g64 routed experts (experts-int4g64/ sidecar) run through its f32 kernel */
#include <pthread.h> /* q38_ehit_mark publishes the lazy HITS table under a lock */
#ifdef COLI_VULKAN
#include "backend_vulkan.h" /* COLI_VULKAN=1: the int8 trunk on a Vulkan device */
@@ -44,12 +45,17 @@ typedef enum {
Q38_WEIGHT_NONE = 0,
Q38_WEIGHT_F32,
Q38_WEIGHT_BF16,
- Q38_WEIGHT_FP8
+ Q38_WEIGHT_FP8,
+ Q38_WEIGHT_INT4G64 /* routed experts from the experts-int4g64/ sidecar */
} Q38WeightKind;
+/* int4-g64 rows: ceil(cols/2) bytes of codes and one f32 scale per 64 inputs */
+static inline int64_t q38_int4_row_bytes(int cols){ return ((int64_t)cols+1)/2; }
+static inline int64_t q38_int4_groups(int cols){ return ((int64_t)cols+XF_BLOCK-1)/XF_BLOCK; }
+
typedef struct {
void *data;
- float *scales; /* block-FP8 only */
+ float *scales; /* block-FP8: per 128x128 block; int4-g64: per row and 64 inputs */
int rows, cols;
int64_t elements, scale_count;
Q38WeightKind kind;
@@ -108,6 +114,8 @@ typedef struct {
uint64_t used;
void *fp8_slab;
int64_t fp8_slab_bytes;
+ void *int4_slab; /* one experts-int4g64/ record: codes, then scales */
+ int64_t int4_slab_bytes;
} Slot;
typedef struct { Slot *slots; int *by_expert, n, cap; } LCache;
@@ -117,6 +125,16 @@ typedef struct {
int ready; /* 0 unknown, 1 resident, -1 incompatible */
} Q38ExpertScaleCache;
+/* The routed experts as int4-g64 (q38_expert_int4_attach): where every
+ * expert's record sits in the sidecar's layer files, and its geometry. */
+typedef struct {
+ shards S; /* <snap>/experts-int4g64/, one file per layer */
+ int *fd; /* [layer] the file holding that layer's records */
+ int64_t *off; /* [layer*experts + expert] where the record starts */
+ int64_t code_bytes[3], scale_count[3]; /* gate, up, down */
+ int64_t scale_off, record_bytes;
+} Q38Int4Experts;
+
typedef enum {
Q38_EXPERT_BATCH_FALLBACK_NONE = 0,
Q38_EXPERT_BATCH_FALLBACK_DISABLED,
@@ -136,6 +154,7 @@ typedef struct {
LCache *cache;
uint8_t **ehit; /* experts routed this turn, for HITS (dashboard Brain) */
Q38ExpertScaleCache *expert_scales;
+ Q38Int4Experts *x4; /* routed experts from experts-int4g64/, NULL = the snapshot's own */
uint64_t clock, hits, miss;
uint64_t expert_weight_reads, expert_scale_reads, expert_pair_reads;
uint64_t expert_prefetch_ranges, expert_parallel_batches;
@@ -263,6 +282,9 @@ static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,int
}
static uint64_t q38_weight_bytes(const Q38Weight *weight) {
+ if(weight->kind==Q38_WEIGHT_INT4G64)
+ return (uint64_t)weight->rows*(uint64_t)q38_int4_row_bytes(weight->cols)+
+ (uint64_t)weight->scale_count*sizeof(float);
uint64_t element_size=weight->kind==Q38_WEIGHT_F32?sizeof(float):
weight->kind==Q38_WEIGHT_BF16?sizeof(uint16_t):
weight->kind==Q38_WEIGHT_FP8?sizeof(uint8_t):0;
@@ -355,6 +377,35 @@ static void q38_matmul_fp8(float *y,const float *x,const uint8_t *q8,const float
matmul_fp8(y,x,q8,bscale,S,I,O);
}
+/* The int4-g64 routed experts (q38_expert_int4_attach) through expert_ffn.h's
+ * f32 kernel: every code is exact in f32, one fma per element and one per
+ * block for its scale, and the activations stay f32 as on the FP8 path. A row
+ * is planar blocks of 64; a width that is not a multiple of 64 (only the tiny
+ * fixtures) keeps its tail in pairs and finishes with the last group's scale.
+ * Each output row is computed the same way whatever S is, so decode and the
+ * prefill batch give the same bits. */
+static void q38_matmul_int4g64(float *y,const float *x,const uint8_t *codes,
+ const float *scales,int S,int I,int O) {
+ int64_t row_bytes=q38_int4_row_bytes(I),groups=q38_int4_groups(I);
+ int body=I/XF_BLOCK*XF_BLOCK;
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o++){
+ const uint8_t *w=codes+(int64_t)o*row_bytes;
+ const float *sc=scales+(int64_t)o*groups;
+ for(int s=0;s<S;s++){
+ const float *xs=x+(int64_t)s*I;
+ float a=body?xf_dot_f32(w,sc,xs,body):0.f;
+ if(body<I){
+ float tail=0.f;
+ for(int i=body;i<I;i++)
+ tail+=(float)((int)((w[i>>1]>>((i&1)*4))&15)-8)*xs[i];
+ a=fmaf(tail,sc[groups-1],a);
+ }
+ y[(int64_t)s*O+o]=a;
+ }
+ }
+}
+
#ifdef COLI_VULKAN
/* COLI_VULKAN=1: a resident matrix answers from the Vulkan device, decode and
* prefill alike, with the weights the CPU would read and in the order
@@ -436,6 +487,8 @@ static void q38_weight_matmul(float *y,const float *x,const Q38Weight *weight,
q38_matmul_bf16(y,x,(const uint16_t*)weight->data,S,I,O);
else if(weight->kind==Q38_WEIGHT_FP8&&weight->scales)
q38_matmul_fp8(y,x,(const uint8_t*)weight->data,weight->scales,S,I,O);
+ else if(weight->kind==Q38_WEIGHT_INT4G64&&weight->scales)
+ q38_matmul_int4g64(y,x,(const uint8_t*)weight->data,weight->scales,S,I,O);
else {fprintf(stderr,"unsupported matmul weight kind %d\n",(int)weight->kind);exit(1);}
}
@@ -1298,11 +1351,22 @@ static int q38_try_load_native_fp8_expert(Model *m,int layer,int expert,Slot *sl
return 1;
}
-static void q38_prefetch_native_fp8_experts(Model *m,int layer,
- const int *experts,int count) {
- if(!m->expert_prefetch||!m->native_fp8||!experts||count<1||
- !q38_prepare_expert_scale_bank(m,layer))return;
+/* Advise the kernel of the experts a route is about to miss: the int4
+ * sidecar's one record each, or the native FP8 pair and down ranges. */
+static void q38_prefetch_experts(Model *m,int layer,const int *experts,int count) {
+ if(!m->expert_prefetch||!experts||count<1)return;
LCache *cache=&m->cache[layer];
+ if(m->x4){
+ for(int index=0;index<count;index++){
+ int expert=experts[index];
+ if(expert<0||expert>=m->c.experts||cache->by_expert[expert]>=0)continue;
+ posix_fadvise(m->x4->fd[layer],m->x4->off[(int64_t)layer*m->c.experts+expert],
+ m->x4->record_bytes,POSIX_FADV_WILLNEED);
+ m->expert_prefetch_ranges++;
+ }
+ return;
+ }
+ if(!m->native_fp8||!q38_prepare_expert_scale_bank(m,layer))return;
for(int index=0;index<count;index++){
int expert=experts[index];st_tensor *weight[3];
if(expert<0||expert>=m->c.experts||cache->by_expert[expert]>=0||
@@ -1381,8 +1445,211 @@ static void q38_load_expert_slice(Model *m,const char *name,const st_tensor *ten
q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;
}
+/* ---- routed experts from the int4-g64 sidecar -------------------------------
+ * tools/convert_qwen38_experts_int4.py writes <snap>/experts-int4g64/: one
+ * safetensors file per layer and an index.json that says what is in them.
+ * The release's FP8 shards are only read by it and stay usable as they are.
+ * Every expert is one contiguous record, so a miss is one read:
+ *
+ * gate codes [I][H/2] | up codes [I][H/2] | down codes [H][I/2] |
+ * gate scales [I][H/64] | up scales [I][H/64] | down scales [H][I/64]
+ *
+ * named like the release's tensors (...experts.E.gate_proj.weight, and the
+ * same name + ".qs" for its f32 scales). The codes are expert_ffn.h's planar
+ * int4 (a row is blocks of 64, byte k holding element k in its low nibble
+ * and element k+32 in its high one, both as v+8; a width that is not a
+ * multiple of 64 keeps its tail in pairs, as idot.h's planarize_i4 leaves
+ * it), one scale per 64 inputs, quantized as tools/convert_qwen36.py --ebits
+ * 4 --gs 64 does. On the release an expert is 2.76 MB, 56% of its 4.92 MB
+ * in FP8: the same RAM holds 1.78x the experts and a miss reads 56% of the
+ * bytes.
+ * Attached by the CLI/serve engine only: the Segment and Edge adapters keep
+ * the snapshot's experts, the representation their numeric class names. */
+#define Q38_INT4_DIR "experts-int4g64"
+#define Q38_INT4_FORMAT "colibri.qwen38.experts-int4g64"
+#define Q38_INT4_VERSION 1
+
+/* Q38_EXPERT_INT4: unset = the sidecar when there is one, 0 = never (the
+ * snapshot's FP8), 1 = refuse to start without it (a benchmark that must not
+ * quietly measure FP8). */
+static int q38_expert_int4_wanted(void) {
+ const char *value=getenv("Q38_EXPERT_INT4");
+ if(!value||!*value)return -1;
+ if(value[0]=='0'&&!value[1])return 0;
+ if(value[0]=='1'&&!value[1])return 1;
+ fprintf(stderr,"Q38_EXPERT_INT4 must be exactly 0 or 1\n");exit(1);
+}
+
+static void q38_int4_refuse(const char *dir,const char *why) {
+ fprintf(stderr,"[qwen38] %s: %s -- refusing (Q38_EXPERT_INT4=0 ignores the sidecar)\n",dir,why);
+ exit(1);
+}
+
+static int64_t q38_json_int(jval *object,const char *key) {
+ jval *value=json_get(object,key);
+ if(!value||value->t!=J_NUM||!isfinite(value->num)||floor(value->num)!=value->num||
+ value->num<0||value->num>(double)INT64_MAX/2)return -1;
+ return (int64_t)value->num;
+}
+
+/* The six tensors of one expert must be the record the index describes:
+ * right dtype and shape, one file, back to back in record order, at an
+ * offset that keeps the f32 scales aligned in a mapping. */
+static int q38_int4_record_at(Model *m,Q38Int4Experts *x,int layer,int expert,
+ int *fd,int64_t *start) {
+ Cfg *c=&m->c;const char *projection[3]={"gate_proj","up_proj","down_proj"};
+ int rows[3]={c->inter,c->inter,c->hidden},cols[3]={c->hidden,c->hidden,c->inter};
+ char suffix[192],name[320];int64_t at=0;
+ for(int part=0;part<6;part++){
+ int k=part%3,is_scale=part>=3;
+ int length=snprintf(suffix,sizeof suffix,"mlp.experts.%d.%s.weight%s",
+ expert,projection[k],is_scale?".qs":"");
+ if(length<0||(size_t)length>=sizeof suffix)return -1;
+ q38_name(m,name,sizeof name,layer,suffix);
+ st_tensor *t=st_find(&x->S,name);
+ int64_t width=is_scale?q38_int4_groups(cols[k]):q38_int4_row_bytes(cols[k]);
+ int64_t bytes=is_scale?x->scale_count[k]*(int64_t)sizeof(float):x->code_bytes[k];
+ if(!t||t->dtype!=(is_scale?2:3)||t->rank!=2||t->shape[0]!=rows[k]||
+ t->shape[1]!=width||t->nbytes!=bytes)return -1;
+ if(!part){
+ if(t->off%(int64_t)sizeof(float))return -1;
+ *fd=t->fd;*start=t->off;
+ }else if(t->fd!=*fd||t->off!=*start+at)return -1;
+ at+=bytes;
+ }
+ return 0;
+}
+
+/* Read <snap>/experts-int4g64/index.json and, if it describes a complete
+ * conversion of this model, index every record of the loaded layers. A
+ * sidecar for another geometry or format, or whose files disagree with the
+ * index, is refused rather than ignored; one still being written is skipped
+ * with a line saying so (or refused under Q38_EXPERT_INT4=1). */
+static void q38_expert_int4_attach(Model *m,const char *snap) {
+ int wanted=q38_expert_int4_wanted();
+ if(!wanted)return;
+ Cfg *c=&m->c;char dir[2100],path[2200];
+ snprintf(dir,sizeof dir,"%s/%s",snap,Q38_INT4_DIR);
+ snprintf(path,sizeof path,"%s/index.json",dir);
+ FILE *f=fopen(path,"rb");
+ if(!f){
+ if(wanted>0){
+ fprintf(stderr,"Q38_EXPERT_INT4=1 but %s cannot be opened "
+ "(tools/convert_qwen38_experts_int4.py writes it)\n",path);exit(1);
+ }
+ return;
+ }
+ char *text=NULL,*arena=NULL;long size=-1;
+ if(!fseek(f,0,SEEK_END))size=ftell(f);
+ if(size<0||size>(16L<<20)||fseek(f,0,SEEK_SET)||!(text=(char*)malloc((size_t)size+1))||
+ fread(text,1,(size_t)size,f)!=(size_t)size){
+ fclose(f);q38_int4_refuse(dir,"index.json cannot be read");
+ }
+ fclose(f);text[size]=0;
+ jval *root=json_parse(text,&arena);
+ if(!root||root->t!=J_OBJ)q38_int4_refuse(dir,"index.json is not a JSON object");
+ const char *format=jstr(root,"format"),*codes=jstr(root,"codes"),*scales=jstr(root,"scales");
+ if(!format||strcmp(format,Q38_INT4_FORMAT)||q38_json_int(root,"version")!=Q38_INT4_VERSION||
+ !codes||strcmp(codes,"planar64-v8")||!scales||strcmp(scales,"f32-per-64")||
+ q38_json_int(root,"group_size")!=XF_BLOCK)
+ q38_int4_refuse(dir,"index.json is not format " Q38_INT4_FORMAT " version 1, int4 codes v+8 "
+ "in planar blocks of 64 with f32 scales per 64");
+ Q38Int4Experts *x=(Q38Int4Experts*)calloc(1,sizeof(*x));
+ if(!x){fprintf(stderr,"OOM int4 expert index\n");exit(1);}
+ int rows[3]={c->inter,c->inter,c->hidden},cols[3]={c->hidden,c->hidden,c->inter};
+ for(int k=0;k<3;k++){
+ x->code_bytes[k]=(int64_t)rows[k]*q38_int4_row_bytes(cols[k]);
+ x->scale_count[k]=(int64_t)rows[k]*q38_int4_groups(cols[k]);
+ x->scale_off+=x->code_bytes[k];
+ }
+ x->record_bytes=x->scale_off+
+ (x->scale_count[0]+x->scale_count[1]+x->scale_count[2])*(int64_t)sizeof(float);
+ if(q38_json_int(root,"layers")!=c->layers||q38_json_int(root,"experts")!=c->experts||
+ q38_json_int(root,"hidden_size")!=c->hidden||
+ q38_json_int(root,"moe_intermediate_size")!=c->inter||
+ q38_json_int(root,"record_bytes")!=x->record_bytes||x->scale_off%(int64_t)sizeof(float))
+ q38_int4_refuse(dir,"index.json was written for another geometry than config.json's");
+ jval *complete=json_get(root,"complete"),*done=json_get(root,"done");
+ if(!complete||complete->t!=J_BOOL||!complete->boolean){
+ int layers_done=done&&done->t==J_ARR?done->len:0;
+ if(wanted>0){
+ fprintf(stderr,"[qwen38] %s: conversion incomplete (%d of %d layers); "
+ "Q38_EXPERT_INT4=1 refuses to fall back to FP8\n",dir,layers_done,c->layers);exit(1);
+ }
+ fprintf(stderr,"[qwen38] %s: conversion incomplete (%d of %d layers); "
+ "the routed experts stay as the snapshot has them\n",dir,layers_done,c->layers);
+ json_free(root);free(arena);free(text);free(x);
+ return;
+ }
+ json_free(root);free(arena);free(text);
+ st_init(&x->S,dir);
+ x->fd=(int*)malloc((size_t)c->layers*sizeof(int));
+ x->off=(int64_t*)malloc((size_t)c->layers*(size_t)c->experts*sizeof(int64_t));
+ if(!x->fd||!x->off){fprintf(stderr,"OOM int4 expert index\n");exit(1);}
+ for(int layer=0;layer<c->layers;layer++){
+ x->fd[layer]=-1;
+ for(int expert=0;expert<c->experts;expert++)x->off[(int64_t)layer*c->experts+expert]=-1;
+ }
+ for(int layer=m->range_begin;layer<m->range_end;layer++)
+ for(int expert=0;expert<c->experts;expert++){
+ int fd=-1;int64_t start=-1;
+ if(q38_int4_record_at(m,x,layer,expert,&fd,&start)||
+ (x->fd[layer]>=0&&fd!=x->fd[layer])){
+ char why[160];
+ snprintf(why,sizeof why,"layer %d expert %d is missing or is not one int4-g64 "
+ "record of %lld bytes",layer,expert,(long long)x->record_bytes);
+ q38_int4_refuse(dir,why);
+ }
+ x->fd[layer]=fd;x->off[(int64_t)layer*c->experts+expert]=start;
+ }
+ m->x4=x;
+}
+
+/* Point a slot's three matrices into one record (a slab or a mapping). */
+static void q38_bind_int4_record(Model *m,Slot *slot,uint8_t *record) {
+ Cfg *c=&m->c;const Q38Int4Experts *x=m->x4;
+ int rows[3]={c->inter,c->inter,c->hidden},cols[3]={c->hidden,c->hidden,c->inter};
+ Q38Weight *dst[3]={&slot->gate,&slot->up,&slot->down};
+ uint8_t *codes=record;float *scales=(float*)(record+x->scale_off);
+ for(int k=0;k<3;k++){
+ Q38Weight *weight=dst[k];q38_weight_free(weight);
+ weight->data=codes;weight->scales=scales;weight->rows=rows[k];weight->cols=cols[k];
+ weight->elements=(int64_t)rows[k]*cols[k];weight->scale_count=x->scale_count[k];
+ weight->kind=Q38_WEIGHT_INT4G64;
+ codes+=x->code_bytes[k];scales+=x->scale_count[k];
+ }
+}
+
+/* One read per miss, into the slot's own slab; with COLI_MAP_EXPERTS=1 the
+ * slot points into the mapped file instead, as the FP8 path does. Safe from
+ * the parallel batch loader: every job owns a distinct slot. */
+static void q38_load_int4_record(Model *m,int layer,int expert,Slot *slot) {
+ Q38Int4Experts *x=m->x4;
+ int fd=x->fd[layer];int64_t off=x->off[(int64_t)layer*m->c.experts+expert];
+ const uint8_t *mapped=(const uint8_t*)st_map_shard_range(fd,off,x->record_bytes);
+ if(mapped){
+ q38_bind_int4_record(m,slot,(uint8_t*)mapped);
+ free(slot->int4_slab);slot->int4_slab=NULL;slot->int4_slab_bytes=0;
+ return;
+ }
+ if(!slot->int4_slab||slot->int4_slab_bytes!=x->record_bytes){
+ void *replacement=realloc(slot->int4_slab,(size_t)x->record_bytes);
+ if(!replacement){fprintf(stderr,"OOM int4 expert record\n");exit(1);}
+ slot->int4_slab=replacement;slot->int4_slab_bytes=x->record_bytes;
+ }
+ q38_bind_int4_record(m,slot,(uint8_t*)slot->int4_slab);
+ st_read_range_raw_cap(&x->S,fd,off,x->record_bytes,slot->int4_slab,
+ slot->int4_slab_bytes,1,"pread Qwen3.8 int4 expert");
+}
+
static void q38_load_expert(Model *m,int layer,int eid,Slot *s) {
Cfg *c=&m->c; int H=c->hidden,I=c->inter; char nm[320],sn[340];
+ if(m->x4){
+ double started=now_s();
+ q38_load_int4_record(m,layer,eid,s);
+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);m->expert_weight_reads++;
+ return;
+ }
q38_name(m,nm,sizeof nm,layer,"mlp.experts.gate_up_proj");
if(st_has(&m->S,nm)){
st_tensor *t=st_find(&m->S,nm);
@@ -1490,7 +1757,7 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,
if(count>cache->cap)
return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_CACHE_CAPACITY,
layer,cache->cap,count);
- if(!q38_prepare_expert_scale_bank(m,layer))
+ if(!m->x4&&!q38_prepare_expert_scale_bank(m,layer))
return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_SCALE_BANK,layer,0,0);
/* The demand set is no longer bounded by the decode top-k: the MoE prefill
* hands over the whole chunk union (up to the cache cap) so its loads run
@@ -1516,7 +1783,7 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,
continue;
}
st_tensor *weight[3];
- if(!q38_native_fp8_expert_tensors(m,layer,expert,weight)){
+ if(!m->x4&&!q38_native_fp8_expert_tensors(m,layer,expert,weight)){
free(jobs);
return q38_expert_batch_fallback(m,Q38_EXPERT_BATCH_FALLBACK_LAYOUT,
layer,expert,0);
@@ -1551,7 +1818,7 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,
if(slot->eid>=0)cache->by_expert[slot->eid]=-1;
}
slot->eid=-1;jobs[job_count].expert=expert;jobs[job_count].slot=slot;
- if(!q38_native_fp8_expert_tensors(m,layer,expert,jobs[job_count].weight)){
+ if(!m->x4&&!q38_native_fp8_expert_tensors(m,layer,expert,jobs[job_count].weight)){
fprintf(stderr,"Qwen3.8 expert layout changed during batch reservation\n");exit(1);
}
job_count++;
@@ -1569,12 +1836,14 @@ static int q38_expert_get_batch(Model *m,int layer,const int *experts,int count,
#endif
double started=now_s();
#pragma omp parallel for schedule(static) num_threads(workers) if(job_count>1)
- for(int job=0;job<job_count;job++)
- q38_load_native_fp8_ranges(m,layer,jobs[job].expert,jobs[job].slot,
- jobs[job].weight);
+ for(int job=0;job<job_count;job++){
+ if(m->x4)q38_load_int4_record(m,layer,jobs[job].expert,jobs[job].slot);
+ else q38_load_native_fp8_ranges(m,layer,jobs[job].expert,jobs[job].slot,
+ jobs[job].weight);
+ }
q38_tm_add(m,Q38_TM_EXPERT_READ,started);
- m->expert_weight_reads+=(uint64_t)job_count*2;
- m->expert_pair_reads+=(uint64_t)job_count;
+ m->expert_weight_reads+=(uint64_t)job_count*(m->x4?1:2);
+ if(!m->x4)m->expert_pair_reads+=(uint64_t)job_count;
if(job_count>1)m->expert_parallel_batches++;
for(int job=0;job<job_count;job++){
Slot *slot=jobs[job].slot;slot->eid=jobs[job].expert;
@@ -2127,6 +2396,13 @@ static void q38_tier_start(Model *m,int cap) {
const char *on=getenv("COLI_CUDA");
if(!on||on[0]!='1'||on[1])return;
Cfg *c=&m->c;
+ if(m->x4){
+ /* The tier streams e4m3 bytes (fmt 8) and copies them at qt_note; its
+ * int4 mode wants every expert resident in RAM. Neither is this. */
+ fprintf(stderr,"[qtier] qwen38: the expert tier streams native FP8 experts only; the int4-g64 "
+ "experts stay on the CPU and the tier stays off (Q38_EXPERT_INT4=0 gives it the FP8 ones)\n");
+ return;
+ }
if(!m->native_fp8){
fprintf(stderr,"[qtier] qwen38: expert tier needs native FP8 experts (Q38_NATIVE_FP8=1); staying on the CPU\n");
return;
@@ -2163,7 +2439,7 @@ static void q38_moe_decode(Model *m,Layer *l,int layer,const float *x,int S,floa
uint32_t qmask=qt_issue(layer,idx,K,xs);
int cpu_idx[Q38_MAX_TOPK],cpu_rank[Q38_MAX_TOPK],cpu_n=0;
for(int z=0;z<K;z++)if(!((qmask>>z)&1u)){cpu_idx[cpu_n]=idx[z];cpu_rank[cpu_n]=z;cpu_n++;}
- q38_prefetch_native_fp8_experts(m,layer,cpu_idx,cpu_n);
+ q38_prefetch_experts(m,layer,cpu_idx,cpu_n);
double phase_started=now_s();
q38_weight_matmul(sg,xs,&l->sh_g,1,H,SI);q38_weight_matmul(su,xs,&l->sh_u,1,H,SI);
for(int j=0;j<SI;j++)sh[j]=q38_silu(sg[j])*su[j];q38_weight_matmul(shared,sh,&l->sh_d,1,SI,H);
@@ -2318,7 +2594,7 @@ static void q38_moe_prefill(Model *m,Layer *l,int layer,const float *x,
}
/* One advice range per distinct expert is enough for this chunk. */
- q38_prefetch_native_fp8_experts(m,layer,unique,unique_count);
+ q38_prefetch_experts(m,layer,unique,unique_count);
/* Shared expert work is independent across rows and remains resident;
* batching it here also keeps its cost out of the routed groups. */
@@ -2610,6 +2886,7 @@ static void q38_model_free(Model *m) {
q38_weight_free(&m->cache[i].slots[s].up);
q38_weight_free(&m->cache[i].slots[s].down);
free(m->cache[i].slots[s].fp8_slab);
+ free(m->cache[i].slots[s].int4_slab);
}
}
free(m->cache[i].slots); free(m->cache[i].by_expert);
@@ -2618,6 +2895,7 @@ static void q38_model_free(Model *m) {
free(m->DN_rec ? m->DN_rec[i] : NULL); free(m->DN_conv ? m->DN_conv[i] : NULL);
free(m->K ? m->K[i] : NULL); free(m->V ? m->V[i] : NULL); free(m->IK ? m->IK[i] : NULL);
}
+ if(m->x4){st_destroy(&m->x4->S);free(m->x4->fd);free(m->x4->off);free(m->x4);}
free(m->L); free(m->cache); free(m->expert_scales); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);
q38_weight_free(&m->embed);q38_weight_free(&m->lm_head);
free(m->final_gr.norm);q38_weight_free(&m->final_gr.down);q38_weight_free(&m->final_gr.up);q38_weight_free(&m->final_gr.inject);
diff --git a/c/tests/test_qwen38_int4_convert.py b/c/tests/test_qwen38_int4_convert.py
new file mode 100644
index 00000000..02f88109
--- /dev/null
+++ b/c/tests/test_qwen38_int4_convert.py
@@ -0,0 +1,402 @@
+"""tools/convert_qwen38_experts_int4.py: the int4-g64 sidecar of Qwen3.8's routed experts.
+
+As a unittest (numpy only, part of `make test`) it converts a synthetic FP8
+snapshot whose widths have partial FP8 blocks and an int4 tail, then reads the
+sidecar back with its own parser and checks, against the source dequantized
+with quant.h's own E4M3_LUT (parsed from the C source):
+ * the layout the engine validates: data on a 4096-byte boundary, every
+ expert one record of six tensors back to back, index.json complete;
+ * the quantizer: every scale is its group's absmax / 7 in f32, every value
+ within half a scale of the source, relative L2 error that of int4-g64;
+ * resume: a second run rewrites nothing, a removed layer is converted again
+ to the same bytes, a changed snapshot is refused.
+
+As a script it verifies an existing sidecar against its snapshot and prints the
+relative L2 error of every expert matrix (sampled with --layers/--experts on
+the real model), and with --engine it checks that qwen38 refuses a sidecar that
+disagrees with its index and falls back from an incomplete one:
+
+ python3 tests/test_qwen38_int4_convert.py --model ./qwen38_tiny_int4 --engine ./qwen38
+"""
+import argparse
+import json
+import math
+import os
+import shutil
+import struct
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+try:
+ import numpy as np
+ HAVE_NUMPY = True
+except ImportError:
+ HAVE_NUMPY = False
+
+HERE = Path(__file__).resolve().parent
+TOOLS = HERE.parent / "tools"
+QUANT_H = HERE.parent / "quant.h"
+PROJ = ("gate_proj", "up_proj", "down_proj")
+
+
+# ---------- independent readers ----------
+
+def engine_e4m3_table():
+ """quant.h's E4M3_LUT, read from the C source the engine compiles."""
+ text = QUANT_H.read_text(encoding="utf-8")
+ body = text[text.index("E4M3_LUT[256] = {") + len("E4M3_LUT[256] = {"):]
+ body = body[:body.index("};")]
+ values = []
+ for item in body.replace("\n", " ").split(","):
+ item = item.strip().rstrip("f")
+ if not item:
+ continue
+ values.append(math.nan if item.upper() == "NAN" else float.fromhex(item))
+ assert len(values) == 256, len(values)
+ return np.array(values, np.float32)
+
+
+def read_safetensors(path):
+ """{name: (dtype, shape, absolute offset, nbytes)} and the data start."""
+ with open(path, "rb") as f:
+ n = struct.unpack("<Q", f.read(8))[0]
+ header = json.loads(f.read(n))
+ tensors = {k: (v["dtype"], v["shape"], 8 + n + v["data_offsets"][0],
+ v["data_offsets"][1] - v["data_offsets"][0])
+ for k, v in header.items() if k != "__metadata__"}
+ return tensors, 8 + n, header.get("__metadata__", {})
+
+
+def read_bytes(path, offset, nbytes):
+ with open(path, "rb") as f:
+ f.seek(offset)
+ return f.read(nbytes)
+
+
+def floats(raw, dtype):
+ if dtype == "BF16":
+ return (np.frombuffer(raw, np.uint16).astype(np.uint32) << 16).view(np.float32)
+ if dtype == "F16":
+ return np.frombuffer(raw, np.float16).astype(np.float32)
+ return np.frombuffer(raw, np.float32)
+
+
+class Snapshot:
+ """The routed experts of a Qwen3.8 snapshot, dequantized as the engine's
+ expanded FP8 path does (byte -> E4M3_LUT, times the f32 block scale)."""
+
+ def __init__(self, model_dir):
+ self.dir = Path(model_dir)
+ config = json.loads((self.dir / "config.json").read_text(encoding="utf-8"))
+ text = config.get("text_config", config)
+ self.layers, self.experts = text["num_hidden_layers"], text["num_experts"]
+ self.hidden, self.inter = text["hidden_size"], text["moe_intermediate_size"]
+ self.where = {}
+ for shard in sorted(self.dir.glob("*.safetensors")):
+ tensors, _, _ = read_safetensors(shard)
+ for name, info in tensors.items():
+ self.where[name] = (shard, info)
+ self.prefix = next(p for p in ("model.language_model", "model")
+ if p + ".embed_tokens.weight" in self.where)
+ self.lut = engine_e4m3_table()
+
+ def matrix(self, layer, expert, k):
+ name = f"{self.prefix}.layers.{layer}.mlp.experts.{expert}.{PROJ[k]}.weight"
+ shard, (dtype, shape, off, n) = self.where[name]
+ rows, cols = shape
+ raw = read_bytes(shard, off, n)
+ if dtype.startswith("F8") or dtype == "float8_e4m3fn":
+ sshard, (sdtype, sshape, soff, sn) = self.where[name + "_scale_inv"]
+ block = floats(read_bytes(sshard, soff, sn), sdtype).reshape(sshape)
+ out = np.empty((rows, cols), np.float32)
+ codes = np.frombuffer(raw, np.uint8).reshape(rows, cols)
+ for r in range(rows):
+ for c0 in range(0, cols, 128):
+ out[r, c0:c0 + 128] = self.lut[codes[r, c0:c0 + 128]] * block[r // 128, c0 // 128]
+ return out
+ return floats(raw, dtype).reshape(rows, cols)
+
+
+def planar_codes(packed_row, cols):
+ """One int4-g64 row's codes (0..15), decoded element by element."""
+ out = np.empty(cols, np.int32)
+ body = cols // 64 * 64
+ for i in range(cols):
+ if i < body:
+ block, k = packed_row[(i // 64) * 32:(i // 64) * 32 + 32], i % 64
+ out[i] = block[k] & 15 if k < 32 else block[k - 32] >> 4
+ else:
+ byte = packed_row[i >> 1]
+ out[i] = byte >> 4 if i & 1 else byte & 15
+ return out
+
+
+class Sidecar:
+ def __init__(self, sidecar_dir):
+ self.dir = Path(sidecar_dir)
+ self.index = json.loads((self.dir / "index.json").read_text(encoding="utf-8"))
+ self.files = {}
+
+ def layer(self, layer):
+ if layer not in self.files:
+ self.files[layer] = read_safetensors(self.dir / f"layer-{layer:03d}.safetensors")
+ return self.files[layer]
+
+ def expert(self, prefix, layer, expert, shapes):
+ """[(codes [R, C] 0..15, scales [R, G])] for gate, up, down, after checking
+ that the six tensors are one record in record order."""
+ tensors, data_start, _ = self.layer(layer)
+ path = self.dir / f"layer-{layer:03d}.safetensors"
+ base = f"{prefix}.layers.{layer}.mlp.experts.{expert}"
+ names = [f"{base}.{p}.weight" for p in PROJ] + [f"{base}.{p}.weight.qs" for p in PROJ]
+ start, at, parts = tensors[names[0]][2], 0, []
+ for i, name in enumerate(names):
+ dtype, shape, off, n = tensors[name]
+ rows, cols = shapes[i % 3]
+ want = ("U8", [rows, (cols + 1) // 2]) if i < 3 else ("F32", [rows, (cols + 63) // 64])
+ if (dtype, shape) != want or off != start + at:
+ raise AssertionError(f"{name}: {dtype} {shape} at +{off - start}, "
+ f"expected {want[0]} {want[1]} at +{at}")
+ parts.append(read_bytes(path, off, n))
+ at += n
+ if at != self.index["record_bytes"] or start % 4:
+ raise AssertionError(f"layer {layer} expert {expert}: record of {at} bytes at {start}")
+ out = []
+ for k in range(3):
+ rows, cols = shapes[k]
+ packed = np.frombuffer(parts[k], np.uint8).reshape(rows, (cols + 1) // 2)
+ codes = np.stack([planar_codes(packed[r], cols) for r in range(rows)])
+ scales = np.frombuffer(parts[3 + k], np.float32).reshape(rows, (cols + 63) // 64)
+ out.append((codes, scales))
+ return out
+
+
+def verify(model_dir, layers=None, experts=None, quiet=False):
+ """Every sampled expert matrix: the quantizer's invariants hold and the
+ relative L2 error against the source. Returns the list of errors."""
+ snap = Snapshot(model_dir)
+ side = Sidecar(snap.dir / "experts-int4g64")
+ idx = side.index
+ assert idx["format"] == "colibri.qwen38.experts-int4g64" and idx["version"] == 1, idx["format"]
+ assert idx["group_size"] == 64 and idx["codes"] == "planar64-v8" and idx["scales"] == "f32-per-64"
+ assert (idx["layers"], idx["experts"], idx["hidden_size"], idx["moe_intermediate_size"]) == \
+ (snap.layers, snap.experts, snap.hidden, snap.inter)
+ assert idx["complete"] is True and idx["done"] == list(range(snap.layers)), "incomplete sidecar"
+ shapes = ((snap.inter, snap.hidden), (snap.inter, snap.hidden), (snap.hidden, snap.inter))
+ errors = []
+ for layer in layers if layers is not None else range(snap.layers):
+ _, data_start, meta = side.layer(layer)
+ assert data_start % 4096 == 0, f"layer {layer}: data starts at {data_start}"
+ assert meta.get("layer") == str(layer), meta
+ for expert in experts if experts is not None else range(snap.experts):
+ stored = side.expert(snap.prefix, layer, expert, shapes)
+ for k, (codes, scales) in enumerate(stored):
+ w = snap.matrix(layer, expert, k)
+ rows, cols = w.shape
+ g = (cols + 63) // 64
+ padded = np.zeros((rows, g * 64), np.float32)
+ padded[:, :cols] = w
+ amax = np.abs(padded.reshape(rows, g, 64)).max(axis=2)
+ expect = np.maximum(amax, np.float32(1e-12)) / np.float32(7.0)
+ if not np.array_equal(scales, expect):
+ raise AssertionError(f"layer {layer} expert {expert} {PROJ[k]}: scales are not absmax/7")
+ values = (codes.astype(np.float32) - 8.0) * np.repeat(scales, 64, axis=1)[:, :cols]
+ slack = np.repeat(scales, 64, axis=1)[:, :cols] * np.float32(0.5 + 1e-5)
+ if not (np.abs(values - w) <= slack).all():
+ raise AssertionError(f"layer {layer} expert {expert} {PROJ[k]}: a value is "
+ f"further than half a scale from its source")
+ err = float(np.linalg.norm((values - w).astype(np.float64)) /
+ max(np.linalg.norm(w.astype(np.float64)), 1e-30))
+ errors.append(err)
+ if not quiet:
+ print(f" layer {layer:3d} expert {expert:3d} {PROJ[k]:9s} rel L2 {err:.4f}")
+ return errors
+
+
+# ---------- a synthetic snapshot ----------
+
+def write_safetensors(path, tensors):
+ header, blobs, offset = {}, [], 0
+ for name, (dtype, shape, raw) in tensors.items():
+ header[name] = {"dtype": dtype, "shape": list(shape), "data_offsets": [offset, offset + len(raw)]}
+ blobs.append(raw)
+ offset += len(raw)
+ encoded = json.dumps(header).encode()
+ encoded += b" " * (-len(encoded) % 8)
+ with open(path, "wb") as f:
+ f.write(struct.pack("<Q", len(encoded)))
+ f.write(encoded)
+ for blob in blobs:
+ f.write(blob)
+
+
+def make_snapshot(root, layers=2, experts=3, hidden=192, inter=72, seed=7):
+ """Per-expert FP8 experts with BF16 scale_inv, gate/up and down in two shards
+ plus the index, as the release ships them. hidden 192 is one and a half FP8
+ blocks and three int4 groups; inter 72 is one group plus a tail of 8."""
+ rng = np.random.default_rng(seed)
+ prefix = "model.language_model"
+ shards = {"model-00001-of-00002.safetensors": {}, "model-00002-of-00002.safetensors": {}}
+ shards["model-00001-of-00002.safetensors"][f"{prefix}.embed_tokens.weight"] = \
+ ("F32", [4, hidden], np.zeros((4, hidden), np.float32).tobytes())
+ for layer in range(layers):
+ for expert in range(experts):
+ for k, (rows, cols) in enumerate(((inter, hidden), (inter, hidden), (hidden, inter))):
+ codes = rng.integers(0, 256, size=(rows, cols), dtype=np.uint8)
+ codes[(codes & 0x7F) == 0x7F] = 0x3C # no NaN encodings
+ scale = (rng.random((-(-rows // 128), -(-cols // 128))) * 0.02 + 0.001).astype(np.float32)
+ bf16 = (scale.view(np.uint32) >> 16).astype(np.uint16)
+ name = f"{prefix}.layers.{layer}.mlp.experts.{expert}.{PROJ[k]}.weight"
+ shard = "model-00002-of-00002.safetensors" if k == 2 else "model-00001-of-00002.safetensors"
+ shards[shard][name] = ("F8_E4M3", [rows, cols], codes.tobytes())
+ shards[shard][name + "_scale_inv"] = ("BF16", list(scale.shape), bf16.tobytes())
+ weight_map = {}
+ for fname, tensors in shards.items():
+ write_safetensors(root / fname, tensors)
+ weight_map.update({name: fname for name in tensors})
+ (root / "model.safetensors.index.json").write_text(json.dumps({"weight_map": weight_map}))
+ (root / "config.json").write_text(json.dumps({"text_config": {
+ "model_type": "qwen4_exp_text", "num_hidden_layers": layers, "num_experts": experts,
+ "hidden_size": hidden, "moe_intermediate_size": inter}}))
+
+
+def import_converter():
+ sys.path.insert(0, str(TOOLS))
+ import convert_qwen38_experts_int4 as converter
+ return converter
+
+
[email protected](HAVE_NUMPY, "numpy is required")
+class ConvertQwen38ExpertsInt4(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self.tmp.name)
+ make_snapshot(self.root)
+ self.converter = import_converter()
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def convert(self, **kw):
+ from contextlib import redirect_stdout
+ import io
+ buf = io.StringIO()
+ with redirect_stdout(buf):
+ self.converter.convert(str(self.root), workers=1, **kw)
+ return buf.getvalue()
+
+ def test_layout_quantizer_and_error(self):
+ self.convert()
+ index = json.loads((self.root / "experts-int4g64" / "index.json").read_text())
+ self.assertTrue(index["complete"])
+ self.assertEqual(index["record_bytes"], 2 * 72 * 96 + 192 * 36 + 4 * (2 * 72 * 3 + 192 * 2))
+ errors = verify(self.root, quiet=True)
+ self.assertEqual(len(errors), 2 * 3 * 3)
+ mean = sum(errors) / len(errors)
+ self.assertTrue(0.05 < mean < 0.15 and max(errors) < 0.2, (mean, max(errors)))
+
+ def test_matches_the_engine_table(self):
+ self.assertTrue(np.array_equal(self.converter.E4M3, engine_e4m3_table(), equal_nan=True))
+
+ def test_resume(self):
+ side = self.root / "experts-int4g64"
+ self.convert(layers="0")
+ index = json.loads((side / "index.json").read_text())
+ self.assertEqual((index["done"], index["complete"]), ([0], False))
+ first = (side / "layer-000.safetensors").read_bytes()
+ mtime = (side / "layer-000.safetensors").stat().st_mtime_ns
+ out = self.convert()
+ self.assertIn("1 layer(s) already converted", out)
+ self.assertEqual((side / "layer-000.safetensors").stat().st_mtime_ns, mtime)
+ self.assertTrue(json.loads((side / "index.json").read_text())["complete"])
+ # a layer whose file is gone is converted again, to the same bytes
+ (side / "layer-000.safetensors").unlink()
+ out = self.convert()
+ self.assertIn("converting it again", out)
+ self.assertEqual((side / "layer-000.safetensors").read_bytes(), first)
+ # a sidecar started from another snapshot is not resumed
+ config = json.loads((self.root / "config.json").read_text())
+ config["note"] = "changed"
+ (self.root / "config.json").write_text(json.dumps(config))
+ with self.assertRaises(SystemExit):
+ self.convert()
+
+
+# ---------- engine checks on a converted fixture ----------
+
+def engine_refusals(model_dir, engine):
+ """qwen38 against edited copies of a converted snapshot."""
+ def run(snap, **env):
+ full = dict(os.environ, OMP_NUM_THREADS="2", SNAP=str(snap), **env)
+ p = subprocess.run([engine, "2", "8", str(snap / "ref.json")], env=full,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ return p.returncode, p.stderr
+
+ def edited(edit_index=None, truncate=False):
+ tmp = Path(tempfile.mkdtemp())
+ snap = tmp / "snap"
+ shutil.copytree(model_dir, snap)
+ index_path = snap / "experts-int4g64" / "index.json"
+ if edit_index:
+ index = json.loads(index_path.read_text())
+ edit_index(index)
+ index_path.write_text(json.dumps(index))
+ if truncate:
+ layer = snap / "experts-int4g64" / "layer-001.safetensors"
+ data = layer.read_bytes()
+ with open(layer, "wb") as f:
+ header = struct.unpack("<Q", data[:8])[0]
+ f.write(data[:8 + header + 64])
+ return tmp, snap
+
+ failures = 0
+ cases = [
+ ("an index for another geometry is refused", dict(edit_index=lambda i: i.update(hidden_size=i["hidden_size"] * 2)), {}, "refusing", True),
+ ("an index of another format version is refused", dict(edit_index=lambda i: i.update(version=2)), {}, "refusing", True),
+ ("a layer file shorter than its header says is refused", dict(truncate=True), {}, "out of file bounds", True),
+ ("an incomplete sidecar falls back to the snapshot's experts", dict(edit_index=lambda i: i.update(complete=False)), {}, "conversion incomplete", False),
+ ("Q38_EXPERT_INT4=1 refuses an incomplete sidecar", dict(edit_index=lambda i: i.update(complete=False)), {"Q38_EXPERT_INT4": "1"}, "refuses to fall back", True),
+ ]
+ for what, edit, env, needle, must_fail in cases:
+ tmp, snap = edited(**edit)
+ try:
+ rc, err = run(snap, **env)
+ finally:
+ shutil.rmtree(tmp, ignore_errors=True)
+ ok = needle in err and ((rc != 0) if must_fail else (rc == 0 and "native FP8" in err))
+ print(f" {'ok ' if ok else 'FAIL'} {what}")
+ failures += not ok
+ return failures
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ ap.add_argument("--model", required=True, help="a snapshot with experts-int4g64/")
+ ap.add_argument("--layers", help="comma list of layers to check (default: all)")
+ ap.add_argument("--experts", help="comma list of experts to check (default: all)")
+ ap.add_argument("--engine", help="qwen38 binary: also check its refusals on edited copies")
+ a = ap.parse_args()
+ layers = [int(x) for x in a.layers.split(",")] if a.layers else None
+ experts = [int(x) for x in a.experts.split(",")] if a.experts else None
+ print(f"{a.model}/experts-int4g64 against the snapshot's experts:")
+ errors = verify(a.model, layers, experts)
+ mean = sum(errors) / len(errors)
+ print(f" {len(errors)} matrices: rel L2 mean {mean:.4f}, max {max(errors):.4f} "
+ f"(every scale absmax/7, every value within half a scale)")
+ failures = 0 if 0.02 < mean < 0.2 and max(errors) < 0.25 else 1
+ if failures:
+ print(" FAIL relative L2 error outside what int4-g64 gives (mean 2-20%, max 25%)")
+ if a.engine:
+ print("qwen38 on edited copies:")
+ failures += engine_refusals(Path(a.model), a.engine)
+ print("test_qwen38_int4_convert: " + ("ok" if not failures else f"{failures} failure(s)"))
+ return 1 if failures else 0
+
+
+if __name__ == "__main__":
+ if len(sys.argv) > 1:
+ sys.exit(main())
+ unittest.main()
diff --git a/c/tests/test_qwen38_int4_experts.c b/c/tests/test_qwen38_int4_experts.c
new file mode 100644
index 00000000..e2af9bc0
--- /dev/null
+++ b/c/tests/test_qwen38_int4_experts.c
@@ -0,0 +1,209 @@
+/* qwen38's routed experts as int4-g64 (the experts-int4g64/ sidecar written by
+ * tools/convert_qwen38_experts_int4.py).
+ *
+ * Pinned with no model:
+ * - q38_weight_matmul on an int4-g64 weight equals the dequantized reference
+ * ((code - 8) times its group's scale, summed in double) to float rounding,
+ * on the release's two expert shapes and on widths whose tail is stored in
+ * pairs, for one row (decode) and for a batch (prefill);
+ * - every row of a batch is the same bits as that row alone, so decode and
+ * the prefill batch cannot drift apart, and a row of whole blocks is the
+ * same bits as expert_ffn.h's scalar reference, its kernel's contract.
+ * With a converted snapshot on argv (make qwen38-tiny-int4-check):
+ * - the sidecar attaches, and every expert loaded through the single LRU path
+ * and through the parallel batch path holds exactly the bytes the sidecar
+ * stores under that expert's tensor names, read by name;
+ * - its matmuls equal the reference built from those bytes;
+ * - the prefetch advises one range per expert the cache does not hold;
+ * - the int4 values are this snapshot's experts: against the engine's own
+ * expanded FP8 their relative L2 error is what int4-g64 costs (2-20%). */
+#define _GNU_SOURCE
+#define QWEN38_NO_MAIN
+#include "../qwen38.c"
+#include "../compat.h" /* setenv/unsetenv: MinGW has neither */
+
+static int fails;
+static void ck(int ok,const char *what){ if(ok){printf(" ok %s\n",what);return;} printf(" FAIL %s\n",what); fails++; }
+
+static unsigned g_seed=20261002;
+static unsigned rnd_u(void){ g_seed=g_seed*1103515245u+12345u; return g_seed>>8; }
+static float rnd(void){ return (float)(rnd_u()&0xFFFF)/32768.f-1.f; }
+
+/* element i of an int4-g64 row, decoded without the engine's helpers */
+static int code_at(const uint8_t *row,int cols,int i){
+ int body=cols/64*64;
+ if(i<body){ const uint8_t *blk=row+(i/64)*32; int k=i%64; return k<32?(blk[k]&15):(blk[k-32]>>4); }
+ return (i&1)?(row[i>>1]>>4):(row[i>>1]&15);
+}
+
+/* y = W x in double, and the sum of |terms| the float error is measured against */
+static void reference(double *y,double *mag,const uint8_t *codes,const float *scales,
+ const float *x,int S,int I,int O){
+ int64_t rb=(I+1)/2,ng=(I+63)/64;
+ for(int s=0;s<S;s++)for(int o=0;o<O;o++){
+ double a=0,m=0;
+ for(int i=0;i<I;i++){
+ double t=(double)(code_at(codes+o*rb,I,i)-8)*(double)scales[o*ng+i/64]*(double)x[(int64_t)s*I+i];
+ a+=t;m+=fabs(t);
+ }
+ y[(int64_t)s*O+o]=a;mag[(int64_t)s*O+o]=m;
+ }
+}
+
+static int within(const float *y,const double *ref,const double *mag,int n,int I){
+ double tol=4.0*(sqrt((double)I)+8.0)*FLT_EPSILON;
+ for(int k=0;k<n;k++)
+ if(!(fabs((double)y[k]-ref[k])<=tol*mag[k]+1e-30))return 0;
+ return 1;
+}
+
+static void kernel_case(int O,int I,int S){
+ int64_t rb=(I+1)/2,ng=(I+63)/64;
+ uint8_t *codes=malloc((size_t)(O*rb));float *scales=malloc((size_t)(O*ng)*sizeof(float));
+ float *x=malloc((size_t)S*I*sizeof(float)),*y=malloc((size_t)S*O*sizeof(float)),*one=malloc((size_t)O*sizeof(float));
+ double *ref=malloc((size_t)S*O*sizeof(double)),*mag=malloc((size_t)S*O*sizeof(double));
+ for(int64_t k=0;k<O*rb;k++)codes[k]=(uint8_t)rnd_u(); /* every nibble, the odd tail's pad too */
+ for(int64_t k=0;k<O*ng;k++)scales[k]=0.001f+0.05f*(rnd()+1.f);
+ for(int64_t k=0;k<(int64_t)S*I;k++)x[k]=rnd()*3.f;
+ Q38Weight w={0};
+ w.data=codes;w.scales=scales;w.rows=O;w.cols=I;w.elements=(int64_t)O*I;
+ w.scale_count=O*ng;w.kind=Q38_WEIGHT_INT4G64;
+ q38_weight_matmul(y,x,&w,S,I,O);
+ reference(ref,mag,codes,scales,x,S,I,O);
+ char what[160];
+ snprintf(what,sizeof what,"[%d x %d], %d row(s): equals the dequantized reference to float rounding",O,I,S);
+ ck(within(y,ref,mag,S*O,I),what);
+ int same=1;
+ for(int s=0;s<S&&same;s++){
+ q38_weight_matmul(one,x+(int64_t)s*I,&w,1,I,O);
+ same=!memcmp(one,y+(int64_t)s*O,(size_t)O*sizeof(float));
+ }
+ snprintf(what,sizeof what,"[%d x %d]: each batch row is the bits of that row alone",O,I);
+ if(S>1)ck(same,what);
+ if(I%64==0){
+ int exact=1;
+ for(int o=0;o<O&&exact;o++)
+ exact=y[o]==xf_dot_f32_ref(codes+o*rb,scales+o*ng,x,I);
+ snprintf(what,sizeof what,"[%d x %d]: bit-identical to expert_ffn.h's scalar reference",O,I);
+ ck(exact,what);
+ }
+ free(codes);free(scales);free(x);free(y);free(one);free(ref);free(mag);
+}
+
+/* the bytes the sidecar stores under this expert's names, read by name */
+static uint8_t *stored(Model *m,int layer,int expert,int k,int scale,int64_t *bytes){
+ const char *projection[3]={"gate_proj","up_proj","down_proj"};
+ char suffix[192],name[320];
+ snprintf(suffix,sizeof suffix,"mlp.experts.%d.%s.weight%s",expert,projection[k],scale?".qs":"");
+ q38_name(m,name,sizeof name,layer,suffix);
+ st_tensor *t=st_find(&m->x4->S,name);
+ if(!t)return NULL;
+ uint8_t *out=malloc((size_t)t->nbytes);
+ st_read_raw(&m->x4->S,name,out,0);*bytes=t->nbytes;
+ return out;
+}
+
+/* a loaded slot holds the stored bytes, and multiplies like them */
+static int slot_matches(Model *m,const Slot *slot,int layer,int expert){
+ const Q38Weight *w[3]={&slot->gate,&slot->up,&slot->down};
+ int ok=slot->eid==expert;
+ for(int k=0;k<3&&ok;k++){
+ int64_t nc=0,ns=0;
+ uint8_t *codes=stored(m,layer,expert,k,0,&nc),*scales=stored(m,layer,expert,k,1,&ns);
+ int O=w[k]->rows,I=w[k]->cols,S=3;
+ ok=codes&&scales&&w[k]->kind==Q38_WEIGHT_INT4G64&&
+ O==(k<2?m->c.inter:m->c.hidden)&&I==(k<2?m->c.hidden:m->c.inter)&&
+ nc==O*q38_int4_row_bytes(I)&&ns==(int64_t)O*q38_int4_groups(I)*(int64_t)sizeof(float)&&
+ !memcmp(w[k]->data,codes,(size_t)nc)&&!memcmp(w[k]->scales,scales,(size_t)ns);
+ if(ok){
+ float *x=malloc((size_t)S*I*sizeof(float)),*y=malloc((size_t)S*O*sizeof(float));
+ double *ref=malloc((size_t)S*O*sizeof(double)),*mag=malloc((size_t)S*O*sizeof(double));
+ for(int64_t q=0;q<(int64_t)S*I;q++)x[q]=rnd();
+ q38_weight_matmul(y,x,w[k],S,I,O);
+ reference(ref,mag,codes,(const float*)scales,x,S,I,O);
+ ok=within(y,ref,mag,S*O,I);
+ free(x);free(y);free(ref);free(mag);
+ }
+ free(codes);free(scales);
+ }
+ return ok;
+}
+
+static double dequant_gap(Model *m,const Slot *fp8,int layer,int expert,int k){
+ const Q38Weight *w=k==0?&fp8->gate:k==1?&fp8->up:&fp8->down;
+ int64_t nc=0,ns=0;
+ uint8_t *codes=stored(m,layer,expert,k,0,&nc);float *scales=(float*)stored(m,layer,expert,k,1,&ns);
+ int O=w->rows,I=w->cols;int64_t rb=(I+1)/2,ng=(I+63)/64;
+ double num=0,den=0;
+ for(int o=0;o<O;o++)for(int i=0;i<I;i++){
+ double source=((const float*)w->data)[(int64_t)o*I+i];
+ double v=(double)(code_at(codes+o*rb,I,i)-8)*scales[o*ng+i/64];
+ num+=(v-source)*(v-source);den+=source*source;
+ }
+ free(codes);free(scales);
+ return den>0?sqrt(num/den):0;
+}
+
+static void fixture_checks(const char *snap){
+ printf("the sidecar of %s\n",snap);
+ setenv("Q38_EXPERT_INT4","1",1); /* refuse to fall back to FP8 */
+ Model m;model_init(&m,snap,1,8);q38_expert_int4_attach(&m,snap);
+ ck(m.x4!=NULL,"the sidecar attaches");
+ if(!m.x4){q38_model_free(&m);return;}
+ int L=m.c.layers,E=m.c.experts,ok=1;
+ for(int layer=0;layer<L;layer++)for(int e=0;e<E;e++)
+ ok&=slot_matches(&m,q38_expert_get(&m,layer,e),layer,e);
+ ck(ok,"single LRU path (cap 1, every expert reloaded): slot bytes are the stored bytes, matmuls equal their reference");
+ ck(m.expert_weight_reads==(uint64_t)L*E&&m.miss==m.expert_weight_reads,"one read per miss");
+ q38_model_free(&m);
+
+ Model b;model_init(&b,snap,E,8);q38_expert_int4_attach(&b,snap);
+ int *all=malloc((size_t)E*sizeof(int));
+ Slot **sel=malloc((size_t)E*sizeof(Slot*));
+ for(int e=0;e<E;e++)all[e]=E-1-e; /* not in id order */
+ q38_prefetch_experts(&b,0,all,E);
+ ck(b.expert_prefetch_ranges==(uint64_t)E,"prefetch advises one range per expert not in the cache");
+ ok=1;int batched=1;
+ for(int layer=0;layer<L;layer++){
+ batched&=q38_expert_get_batch(&b,layer,all,E,sel);
+ for(int e=0;e<E&&batched;e++)ok&=slot_matches(&b,sel[e],layer,all[e]);
+ }
+ ck(batched&&b.expert_parallel_batches==(uint64_t)L,"parallel batch path taken for every layer");
+ ck(ok,"batch path: slot bytes are the stored bytes, matmuls equal their reference");
+ uint64_t prefetched=b.expert_prefetch_ranges;
+ q38_prefetch_experts(&b,0,all,E);
+ ck(b.expert_prefetch_ranges==prefetched,"nothing is advised once the experts are cached");
+
+ /* the same experts from the snapshot itself, expanded to f32 by the engine */
+ setenv("Q38_EXPERT_INT4","0",1);setenv("Q38_NATIVE_FP8","0",1);setenv("Q38_NATIVE_BF16","0",1);
+ Model f;model_init(&f,snap,1,8);q38_expert_int4_attach(&f,snap);
+ ck(f.x4==NULL,"Q38_EXPERT_INT4=0 leaves the snapshot's experts");
+ double worst=0,sum=0;int n=0,kind_ok=1;
+ for(int layer=0;layer<L;layer++)for(int e=0;e<E;e++){
+ Slot *slot=q38_expert_get(&f,layer,e);
+ kind_ok&=slot->gate.kind==Q38_WEIGHT_F32;
+ for(int k=0;k<3;k++){double g=dequant_gap(&b,slot,layer,e,k);sum+=g;n++;if(g>worst)worst=g;}
+ }
+ printf(" int4-g64 against the engine's expanded experts: relative L2 mean %.4f, worst %.4f over %d matrices\n",
+ n?sum/n:0,worst,n);
+ ck(kind_ok&&n&&sum/n>0.02&&worst<0.2,"the int4 values are this snapshot's experts, within int4-g64's error");
+ unsetenv("Q38_EXPERT_INT4");unsetenv("Q38_NATIVE_FP8");unsetenv("Q38_NATIVE_BF16");
+ q38_model_free(&f);free(all);free(sel);q38_model_free(&b);
+}
+
+int main(int argc,char **argv){
+ printf("the int4-g64 matmul\n");
+ kernel_case(640,2560,1); /* gate/up of the release */
+ kernel_case(640,2560,5);
+ kernel_case(2560,640,1); /* down */
+ kernel_case(2560,640,9);
+ kernel_case(8,32,3); /* the tiny fixture: one short group */
+ kernel_case(32,8,3);
+ kernel_case(7,130,4); /* two blocks and a two-element tail */
+ kernel_case(5,99,2); /* an odd tail: the last byte's high nibble is padding */
+ if(argc>1)fixture_checks(argv[1]);
+ else printf("(no snapshot on argv: the sidecar checks run from make qwen38-tiny-int4-check)\n");
+ if(fails){printf("test_qwen38_int4_experts: %d failure(s)\n",fails);return 1;}
+ printf("test_qwen38_int4_experts: ok\n");
+ return 0;
+}
diff --git a/c/tests/test_qwen38_tier_engine.c b/c/tests/test_qwen38_tier_engine.c
index c439df1e..d6020df4 100644
--- a/c/tests/test_qwen38_tier_engine.c
+++ b/c/tests/test_qwen38_tier_engine.c
@@ -10,9 +10,13 @@
* oracle tokens/logits must still match exactly; the hit/miss counters pin
* that residency was consulted anyway. Real-GPU parity is a hardware run.
*
- * Needs ./qwen38_tiny_fp8 (make qwen38-tiny-fp8-generate). Include order as
- * in test_qwen36_tier_int8_engine.c: engine first, then the fake backend,
- * then the tier source, so the tier's statics (G, qs()) are readable here. */
+ * Needs ./qwen38_tiny_fp8 (make qwen38-tiny-fp8-generate). Given a snapshot
+ * converted to int4-g64 experts on argv (make qwen38-tiny-int4-check passes
+ * ./qwen38_tiny_int4), a third pass checks that the tier declines those
+ * experts -- it streams e4m3 bytes only -- and that the engine then computes
+ * them on the CPU and reproduces the int4 reference. Include order as in
+ * test_qwen36_tier_int8_engine.c: engine first, then the fake backend, then
+ * the tier source, so the tier's statics (G, qs()) are readable here. */
#define main qwen38_main_unused
#include "../qwen38.c"
#undef main
@@ -34,7 +38,7 @@ static void tk(int ok, const char *what) {
t_fails++;
}
-int main(void) {
+int main(int argc, char **argv_in) {
/* the fixture: 4 layers x 4 experts, hidden 32, inter 8, top-2, all
* routed experts native e4m3 with one 128x128 block scale per matrix */
setenv("SNAP", "./qwen38_tiny_fp8", 1);
@@ -113,6 +117,26 @@ int main(void) {
qt_shutdown();
tk(qt_dense_count() == 0, "dense handles released at shutdown");
+ if (argc > 1) {
+ /* Third pass: the int4-g64 sidecar. The tier must stay off -- no
+ * upload, no routed expert ever consulted -- and the CPU must answer
+ * every expert from the sidecar, matching the int4 reference. */
+ printf(" int4-g64 experts (%s)\n", argv_in[1]);
+ char ref[2048]; snprintf(ref, sizeof ref, "%s/ref_int4.json", argv_in[1]);
+ char *argv4[] = { (char *)"qwen38", (char *)"2", (char *)"8", ref };
+ setenv("SNAP", argv_in[1], 1);
+ setenv("Q38_EXPERT_INT4", "1", 1);
+ fake_uploads = 0; fake_dense_compute = 0;
+ unsigned long long before = G.miss; for (int i = 0; i < QT_MAX_DEV; i++) before += G.hits[i];
+ rc = qwen38_main_unused(4, argv4);
+ tk(rc == 0, "engine with int4 experts under COLI_CUDA=1 reproduces the int4 reference");
+ tk(!G.on && !G_fp8_stream, "the tier declined: it streams native FP8 experts only");
+ tk(fake_uploads == 0 && qt_dense_count() == 0, "nothing was uploaded, experts or trunk");
+ unsigned long long consulted = G.miss; for (int i = 0; i < QT_MAX_DEV; i++) consulted += G.hits[i];
+ tk(consulted == before, "qt_issue never took a routed expert off the CPU");
+ unsetenv("Q38_EXPERT_INT4");
+ }
+
if (t_fails) { printf("test_qwen38_tier_engine: %d failure(s)\n", t_fails); return 1; }
printf("test_qwen38_tier_engine: ok\n");
return 0;
diff --git a/c/tools/convert_qwen38_experts_int4.py b/c/tools/convert_qwen38_experts_int4.py
new file mode 100644
index 00000000..ac65901a
--- /dev/null
+++ b/c/tools/convert_qwen38_experts_int4.py
@@ -0,0 +1,551 @@
+#!/usr/bin/env python3
+"""Qwen3.8: the routed experts as int4-g64, in a sidecar next to the release.
+
+The release stores every routed expert as FP8 e4m3 with one scale per 128x128
+block, 4.9 MB per expert on Qwen3.8-Flash-Next. Decode waits on those bytes:
+an expert that is not in the RAM cache is read from disk, and how many fit in
+the cache decides how often that happens. This converter writes the same
+experts as int4 with one f32 scale per 64 inputs, 2.76 MB per expert, into
+<model>/experts-int4g64/. The FP8 shards are only read: the snapshot stays
+usable as it is, and qwen38 picks the sidecar up when it is there
+(Q38_EXPERT_INT4=0 ignores it).
+
+Only the routed experts change. The shared expert, attention, DeltaNet, PLE,
+vision and MTP tensors are not read at all.
+
+Numerics, so the engine and every tier read the bytes they already know:
+ dequantize e4m3 byte -> quant.h's E4M3_LUT value, times the block's
+ weight_scale_inv (BF16 or F32, widened to f32), one f32 multiply:
+ exactly the expanded FP8 path in qwen38_core.h. BF16/F16/F32
+ experts (the tiny fixture, fused or per expert) are widened.
+ quantize tools/convert_qwen36.py's quantize_row_grouped with --ebits 4
+ --gs 64: per row, groups of 64 inputs (a short last group is
+ zero-padded), scale = max(absmax, 1e-12) / 7 in f32, code =
+ round-half-even(w / scale) clamped to [-8, 7].
+ codes stored as v+8 in expert_ffn.h's planar layout: a row is blocks
+ of 64 elements, 32 bytes each, byte k holding element k in its
+ low nibble and element k+32 in its high nibble. A width that is
+ not a multiple of 64 keeps its tail in pairs (byte j = element
+ 2j low, 2j+1 high), as idot.h's planarize_i4 leaves it. A row is
+ ceil(cols/2) bytes.
+ scales f32, [rows][ceil(cols/64)].
+
+Layout: one safetensors file per layer, layer-NNN.safetensors, and index.json.
+Every expert is one contiguous record, so a cache miss is one read:
+
+ gate codes U8 [I, H/2] | up codes U8 [I, H/2] | down codes U8 [H, I/2] |
+ gate scales F32 [I, H/64] | up scales F32 [I, H/64] | down scales F32 [H, I/64]
+
+named like the release's tensors (<prefix>.layers.L.mlp.experts.E.gate_proj.weight
+and the same name + ".qs" for its scales, the .qs convention of the other
+int4 containers). Records follow each other with no gap; the data region of
+a file starts on a 4096-byte boundary, so on the release's geometry (record
+2,764,800 bytes = 675 pages) every record is page aligned.
+
+index.json carries the format name and version, the geometry, the group size,
+the record size and the layers already written. It is rewritten (atomically)
+after every layer and says "complete": true only when all of them are there;
+the engine refuses anything else. A layer is written to a .part file and
+renamed when it is whole, so an interrupted run resumes at the first layer
+the index does not list.
+
+Usage:
+ python3 tools/convert_qwen38_experts_int4.py --model /path/to/qwen38-fp8 --plan
+ python3 tools/convert_qwen38_experts_int4.py --model /path/to/qwen38-fp8 [--workers N]
+
+Peak RAM does not grow with the model: each worker holds one expert in flight
+(about 0.11 GB with numpy), and the writer at most the records of one layer
+(1.4 GB) when the disk lags behind the workers. Measured on a 12-thread
+i7-1355U with a synthetic snapshot of the release's expert geometry: 22 s of
+compute per layer with 12 workers.
+"""
+import argparse
+import hashlib
+import json
+import math
+import os
+import struct
+import sys
+import time
+
+import numpy as np
+
+try:
+ sys.stdout.reconfigure(errors="backslashreplace")
+except AttributeError:
+ pass
+
+SIDECAR_DIR = "experts-int4g64"
+FORMAT = "colibri.qwen38.experts-int4g64"
+VERSION = 1
+GS = 64
+CODES = "planar64-v8"
+SCALES = "f32-per-64"
+DATA_ALIGN = 4096
+PROJ = ("gate_proj", "up_proj", "down_proj")
+FP8_BLOCK = 128
+
+_FLOAT_DTYPES = {"BF16": 2, "F16": 2, "F32": 4}
+_FP8_DTYPES = ("F8_E4M3", "F8_E4M3FN", "float8_e4m3fn")
+
+
+def e4m3_table():
+ """quant.h's E4M3_LUT (torch.float8_e4m3fn): bias 7, subnormals at
+ exponent 0, only 0x7F/0xFF are NaN (no infinities)."""
+ table = np.empty(256, np.float32)
+ for byte in range(256):
+ exponent, mantissa = (byte >> 3) & 0xF, byte & 7
+ if exponent == 0xF and mantissa == 7:
+ value = math.nan
+ elif exponent == 0:
+ value = mantissa / 8.0 * 2.0 ** -6
+ else:
+ value = (1.0 + mantissa / 8.0) * 2.0 ** (exponent - 7)
+ table[byte] = -value if byte & 0x80 else value
+ return table
+
+
+E4M3 = e4m3_table()
+
+
+# ---------- int4-g64: quantize, pack, and their inverses ----------
+
+def groups(cols):
+ return (cols + GS - 1) // GS
+
+
+def row_bytes(cols):
+ return (cols + 1) // 2
+
+
+def quantize_g64(w):
+ """w f32 [R, C] -> (codes u8 [R, C] holding v+8, scales f32 [R, ceil(C/64)]).
+ tools/convert_qwen36.py quantize_row_grouped, bits=4, in numpy."""
+ rows, cols = w.shape
+ ng = groups(cols)
+ pad = ng * GS - cols
+ padded = np.pad(w, ((0, 0), (0, pad))) if pad else w
+ g = padded.reshape(rows, ng, GS)
+ amax = np.abs(g).max(axis=2, keepdims=True)
+ scales = np.maximum(amax, np.float32(1e-12)) / np.float32(7.0)
+ codes = np.clip(np.rint(g / scales), -8, 7).astype(np.int16) + 8
+ return codes.reshape(rows, ng * GS)[:, :cols].astype(np.uint8), scales.reshape(rows, ng)
+
+
+def pack_planar(codes):
+ """u8 codes [R, C] (0..15) -> u8 [R, ceil(C/2)]: planar blocks of 64, tail in pairs."""
+ rows, cols = codes.shape
+ body = cols // GS * GS
+ parts = []
+ if body:
+ blocks = codes[:, :body].reshape(rows, body // GS, 2, GS // 2)
+ parts.append((blocks[:, :, 0, :] | (blocks[:, :, 1, :] << 4)).reshape(rows, body // 2))
+ if body < cols:
+ tail = codes[:, body:]
+ if tail.shape[1] & 1:
+ tail = np.concatenate([tail, np.full((rows, 1), 8, np.uint8)], axis=1)
+ parts.append(tail[:, 0::2] | (tail[:, 1::2] << 4))
+ return np.ascontiguousarray(np.concatenate(parts, axis=1) if len(parts) > 1 else parts[0])
+
+
+def unpack_planar(packed, cols):
+ """Inverse of pack_planar: u8 [R, ceil(C/2)] -> codes u8 [R, C]."""
+ rows = packed.shape[0]
+ body = cols // GS * GS
+ out = np.empty((rows, cols), np.uint8)
+ if body:
+ blocks = packed[:, :body // 2].reshape(rows, body // GS, GS // 2)
+ out[:, :body] = np.stack([blocks & 15, blocks >> 4], axis=2).reshape(rows, body)
+ if body < cols:
+ tail = packed[:, body // 2:]
+ pairs = np.stack([tail & 15, tail >> 4], axis=2).reshape(rows, -1)
+ out[:, body:] = pairs[:, :cols - body]
+ return out
+
+
+def dequantize(codes, scales):
+ """codes u8 [R, C], scales f32 [R, ceil(C/64)] -> f32 [R, C]."""
+ cols = codes.shape[1]
+ return (codes.astype(np.float32) - 8.0) * np.repeat(scales, GS, axis=1)[:, :cols]
+
+
+def rel_l2(reference, approx):
+ """||reference - approx|| / ||reference||, in float64. Plain reductions, not
+ np.linalg.norm: that is a BLAS call, and every worker's BLAS starting a
+ thread per core made a 12-worker layer 2.4x slower (46 s against 19 s)."""
+ ref = reference.astype(np.float64)
+ norm = float(np.sum(np.square(ref)))
+ diff = float(np.sum(np.square(ref - approx.astype(np.float64))))
+ return math.sqrt(diff / norm) if norm > 0 else 0.0
+
+
+# ---------- the geometry of one record ----------
+
+class Geometry:
+ def __init__(self, layers, experts, hidden, inter):
+ self.layers, self.experts, self.hidden, self.inter = layers, experts, hidden, inter
+ self.shape = ((inter, hidden), (inter, hidden), (hidden, inter))
+ self.code_bytes = [r * row_bytes(c) for r, c in self.shape]
+ self.scale_count = [r * groups(c) for r, c in self.shape]
+ self.scale_off = sum(self.code_bytes)
+ self.record_bytes = self.scale_off + 4 * sum(self.scale_count)
+ if self.scale_off % 4:
+ raise ValueError(f"hidden={hidden}, moe_intermediate_size={inter}: the codes of an "
+ f"expert are {self.scale_off} bytes, not a multiple of 4, so its f32 "
+ f"scales could not follow them aligned; this geometry is not supported")
+
+ def tensors(self, prefix, layer, expert):
+ """(name, dtype, shape, offset in the record, bytes) in record order."""
+ base = f"{prefix}.layers.{layer}.mlp.experts.{expert}"
+ out, off = [], 0
+ for k, (r, c) in enumerate(self.shape):
+ out.append((f"{base}.{PROJ[k]}.weight", "U8", [r, row_bytes(c)], off, self.code_bytes[k]))
+ off += self.code_bytes[k]
+ for k, (r, c) in enumerate(self.shape):
+ n = 4 * self.scale_count[k]
+ out.append((f"{base}.{PROJ[k]}.weight.qs", "F32", [r, groups(c)], off, n))
+ off += n
+ assert off == self.record_bytes
+ return out
+
+ def fp8_expert_bytes(self):
+ return sum(r * c for r, c in self.shape)
+
+
+# ---------- the release, read by offset ----------
+
+def read_header(path):
+ with open(path, "rb") as f:
+ n = struct.unpack("<Q", f.read(8))[0]
+ if n > 512 << 20:
+ raise ValueError(f"{path}: safetensors header of {n} bytes")
+ return json.loads(f.read(n)), 8 + n
+
+
+class Source:
+ """The routed experts of a Qwen3.8 snapshot, located by name in its shards."""
+
+ def __init__(self, model_dir):
+ self.dir = model_dir
+ with open(os.path.join(model_dir, "config.json"), "rb") as f:
+ self.config_bytes = f.read()
+ config = json.loads(self.config_bytes)
+ text = config.get("text_config", config)
+ if text.get("model_type") != "qwen4_exp_text":
+ sys.exit(f"{model_dir}: text model_type {text.get('model_type')!r} is not qwen4_exp_text")
+ self.geometry = Geometry(int(text["num_hidden_layers"]), int(text["num_experts"]),
+ int(text["hidden_size"]), int(text["moe_intermediate_size"]))
+ index_path = os.path.join(model_dir, "model.safetensors.index.json")
+ if os.path.exists(index_path):
+ with open(index_path, "rb") as f:
+ weight_map = json.load(f)["weight_map"]
+ files = sorted(set(weight_map.values()))
+ else:
+ weight_map = None
+ files = sorted(f for f in os.listdir(model_dir) if f.endswith(".safetensors"))
+ if not files:
+ sys.exit(f"{model_dir}: no safetensors shards")
+ self.tensors = {}
+ for name in files:
+ path = os.path.join(model_dir, name)
+ header, base = read_header(path)
+ for key, meta in header.items():
+ if key == "__metadata__" or (weight_map and weight_map.get(key) != name):
+ continue
+ if ".mlp.experts." not in key:
+ continue
+ begin, end = meta["data_offsets"]
+ self.tensors[key] = (path, base + begin, end - begin, meta["dtype"], meta["shape"])
+ if os.path.exists(index_path):
+ prefixes = [p for p in ("model.language_model", "model")
+ if p + ".embed_tokens.weight" in weight_map]
+ else:
+ prefixes = []
+ for name in files:
+ header, _ = read_header(os.path.join(model_dir, name))
+ prefixes += [p for p in ("model.language_model", "model")
+ if p + ".embed_tokens.weight" in header]
+ if not prefixes:
+ sys.exit(f"{model_dir}: no Qwen4-Exp text embedding; is this a Qwen3.8 snapshot?")
+ self.prefix = prefixes[0]
+ self._fds = {}
+
+ def fingerprint(self):
+ """What a resumed run must find unchanged: the config and the shards
+ that hold routed experts (name and size)."""
+ shards = sorted({t[0] for t in self.tensors.values()})
+ return {"config_sha256": hashlib.sha256(self.config_bytes).hexdigest(),
+ "shards": {os.path.basename(p): os.path.getsize(p) for p in shards}}
+
+ def _read(self, path, offset, nbytes):
+ fd = self._fds.get(path)
+ if fd is None:
+ fd = self._fds[path] = os.open(path, os.O_RDONLY | getattr(os, "O_BINARY", 0))
+ out = bytearray()
+ while len(out) < nbytes:
+ chunk = os.pread(fd, nbytes - len(out), offset + len(out))
+ if not chunk:
+ raise IOError(f"{path}: short read at {offset + len(out)}")
+ out += chunk
+ return bytes(out)
+
+ def _float(self, entry, element_offset, count):
+ path, offset, nbytes, dtype, _ = entry
+ size = _FLOAT_DTYPES[dtype]
+ raw = self._read(path, offset + element_offset * size, count * size)
+ if dtype == "BF16":
+ return (np.frombuffer(raw, np.uint16).astype(np.uint32) << 16).view(np.float32)
+ if dtype == "F16":
+ return np.frombuffer(raw, np.float16).astype(np.float32)
+ return np.frombuffer(raw, np.float32).copy()
+
+ def matrix(self, layer, expert, k):
+ """Expert `expert`'s projection k (gate, up, down) as the engine sees it, f32."""
+ rows, cols = self.geometry.shape[k]
+ name = f"{self.prefix}.layers.{layer}.mlp.experts.{expert}.{PROJ[k]}.weight"
+ entry = self.tensors.get(name)
+ if entry is not None:
+ path, offset, nbytes, dtype, shape = entry
+ if list(shape) != [rows, cols]:
+ raise ValueError(f"{name}: shape {shape}, expected {[rows, cols]}")
+ if dtype in _FP8_DTYPES:
+ if nbytes != rows * cols:
+ raise ValueError(f"{name}: {nbytes} bytes for an e4m3 [{rows}, {cols}]")
+ scale = self.tensors.get(name + "_scale_inv")
+ nb_r, nb_c = -(-rows // FP8_BLOCK), -(-cols // FP8_BLOCK)
+ if scale is None or list(scale[4]) != [nb_r, nb_c] or scale[3] not in _FLOAT_DTYPES:
+ raise ValueError(f"{name}_scale_inv: missing, or not a [{nb_r}, {nb_c}] float tensor")
+ block = self._float(scale, 0, nb_r * nb_c).reshape(nb_r, nb_c)
+ codes = np.frombuffer(self._read(path, offset, nbytes), np.uint8).reshape(rows, cols)
+ expand = np.repeat(np.repeat(block, FP8_BLOCK, axis=0)[:rows], FP8_BLOCK, axis=1)[:, :cols]
+ w = E4M3[codes] * expand
+ elif dtype in _FLOAT_DTYPES:
+ w = self._float(entry, 0, rows * cols).reshape(rows, cols)
+ else:
+ raise ValueError(f"{name}: unsupported expert dtype {dtype}")
+ else:
+ fused = "gate_up_proj" if k < 2 else "down_proj"
+ fname = f"{self.prefix}.layers.{layer}.mlp.experts.{fused}"
+ entry = self.tensors.get(fname)
+ g = self.geometry
+ want = [g.experts, 2 * g.inter, g.hidden] if k < 2 else [g.experts, g.hidden, g.inter]
+ if entry is None:
+ raise ValueError(f"layer {layer} expert {expert}: neither {name} nor {fname}")
+ if list(entry[4]) != want or entry[3] not in _FLOAT_DTYPES:
+ raise ValueError(f"{fname}: expected a float {want}, found {entry[3]} {entry[4]}")
+ first = expert * want[1] * want[2] + (rows * cols if k == 1 else 0)
+ w = self._float(entry, first, rows * cols).reshape(rows, cols)
+ if not np.isfinite(w).all():
+ raise ValueError(f"{name}: non-finite weights; refusing to quantize them")
+ return w
+
+
+# ---------- one expert, one layer ----------
+
+_SOURCE = None
+
+
+def _init_worker(model_dir):
+ global _SOURCE
+ _SOURCE = Source(model_dir)
+
+
+def convert_expert(task):
+ """(layer, expert) -> (the record's bytes, relative L2 error of gate, up,
+ down, this process's peak RSS in MB)."""
+ layer, expert = task
+ codes, scales, errors = [], [], []
+ for k in range(3):
+ w = _SOURCE.matrix(layer, expert, k)
+ q, s = quantize_g64(w)
+ errors.append(rel_l2(w, dequantize(q, s)))
+ codes.append(pack_planar(q).tobytes())
+ scales.append(s.astype(np.float32).tobytes())
+ return b"".join(codes) + b"".join(scales), errors, peak_rss_mb()
+
+
+def layer_file(layer):
+ return f"layer-{layer:03d}.safetensors"
+
+
+def layer_header(geometry, prefix, layer):
+ """The layer file's header, padded so the data region starts on DATA_ALIGN."""
+ header, meta = {}, {"format": FORMAT, "version": str(VERSION), "layer": str(layer)}
+ for expert in range(geometry.experts):
+ base = expert * geometry.record_bytes
+ for name, dtype, shape, off, n in geometry.tensors(prefix, layer, expert):
+ header[name] = {"dtype": dtype, "shape": shape, "data_offsets": [base + off, base + off + n]}
+ header["__metadata__"] = meta
+ raw = json.dumps(header, separators=(",", ":")).encode()
+ raw += b" " * ((-(8 + len(raw))) % DATA_ALIGN)
+ return struct.pack("<Q", len(raw)) + raw
+
+
+def layer_file_bytes(geometry, prefix, layer):
+ return len(layer_header(geometry, prefix, layer)) + geometry.experts * geometry.record_bytes
+
+
+def write_layer(src, out_dir, layer, pool):
+ g = src.geometry
+ final = os.path.join(out_dir, layer_file(layer))
+ part = final + ".part"
+ tasks = [(layer, e) for e in range(g.experts)]
+ errors, worker_mb = [], 0.0
+ with open(part, "wb") as f:
+ f.write(layer_header(g, src.prefix, layer))
+ results = pool.imap(convert_expert, tasks, chunksize=4) if pool else map(convert_expert, tasks)
+ for record, err, rss in results:
+ if len(record) != g.record_bytes:
+ raise RuntimeError(f"layer {layer}: record of {len(record)} bytes, expected {g.record_bytes}")
+ f.write(record)
+ errors.append(err)
+ worker_mb = max(worker_mb, rss)
+ f.flush()
+ os.fsync(f.fileno())
+ os.replace(part, final)
+ return np.array(errors), worker_mb
+
+
+def write_index(out_dir, index):
+ tmp = os.path.join(out_dir, "index.json.part")
+ with open(tmp, "w", encoding="utf-8") as f:
+ json.dump(index, f, indent=1)
+ f.write("\n")
+ f.flush()
+ os.fsync(f.fileno())
+ os.replace(tmp, os.path.join(out_dir, "index.json"))
+
+
+def new_index(src):
+ g = src.geometry
+ return {
+ "format": FORMAT, "version": VERSION, "complete": False,
+ "group_size": GS, "codes": CODES, "scales": SCALES,
+ "layers": g.layers, "experts": g.experts,
+ "hidden_size": g.hidden, "moe_intermediate_size": g.inter,
+ "prefix": src.prefix, "record_bytes": g.record_bytes,
+ "record": ["gate codes", "up codes", "down codes", "gate scales", "up scales", "down scales"],
+ "quantizer": "per 64 inputs: scale = max(absmax, 1e-12)/7 (f32), code = rint(w/scale) "
+ "clamped to [-8,7], stored v+8 (tools/convert_qwen36.py quantize_row_grouped)",
+ "files": [layer_file(i) for i in range(g.layers)],
+ "done": [],
+ "source": src.fingerprint(),
+ }
+
+
+def resume_index(out_dir, src):
+ """The index of a previous run on the same snapshot, or a fresh one."""
+ path = os.path.join(out_dir, "index.json")
+ fresh = new_index(src)
+ if not os.path.exists(path):
+ return fresh
+ with open(path, encoding="utf-8") as f:
+ index = json.load(f)
+ keys = ("format", "version", "group_size", "codes", "scales", "layers", "experts",
+ "hidden_size", "moe_intermediate_size", "prefix", "record_bytes", "source")
+ stale = [k for k in keys if index.get(k) != fresh[k]]
+ if stale:
+ sys.exit(f"{path} was written for something else ({', '.join(stale)} differ); "
+ f"remove {out_dir} to convert this snapshot from scratch")
+ done = []
+ for layer in index.get("done", []):
+ path = os.path.join(out_dir, layer_file(layer))
+ if os.path.exists(path) and os.path.getsize(path) == layer_file_bytes(src.geometry, src.prefix, layer):
+ done.append(layer)
+ else:
+ print(f"layer {layer}: listed as done but {path} is missing or the wrong size; converting it again")
+ fresh["done"] = sorted(done)
+ fresh["complete"] = len(done) == src.geometry.layers
+ return fresh
+
+
+def parse_layers(spec, count):
+ if not spec:
+ return list(range(count))
+ out = set()
+ for part in spec.split(","):
+ a, _, b = part.partition("-")
+ lo, hi = int(a), int(b) if b else int(a)
+ if not 0 <= lo <= hi < count:
+ sys.exit(f"--layers {spec}: {part} is outside 0..{count - 1}")
+ out.update(range(lo, hi + 1))
+ return sorted(out)
+
+
+def peak_rss_mb():
+ """This process's peak resident set in MB (0 where getrusage is missing)."""
+ try:
+ import resource
+ except ImportError:
+ return 0.0
+ peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+ return peak / (1 << 20) if sys.platform == "darwin" else peak / 1024
+
+
+def convert(model_dir, out_dir=None, workers=1, layers=None, plan=False):
+ """Write (or finish) model_dir's sidecar; returns its directory."""
+ src = Source(model_dir)
+ g = src.geometry
+ out_dir = out_dir or os.path.join(model_dir, SIDECAR_DIR)
+ total = sum(layer_file_bytes(g, src.prefix, i) for i in range(g.layers))
+ fp8 = g.fp8_expert_bytes() * g.experts * g.layers
+ print(f"{model_dir}: {g.layers} layers x {g.experts} experts, hidden {g.hidden}, "
+ f"moe_intermediate {g.inter}, prefix {src.prefix}")
+ print(f" per expert: FP8 {g.fp8_expert_bytes():,} bytes -> int4-g64 {g.record_bytes:,} bytes "
+ f"({100 * g.record_bytes / g.fp8_expert_bytes():.1f}%)")
+ print(f" sidecar {out_dir}: {total:,} bytes ({total / 2**30:.2f} GiB) "
+ f"for {fp8:,} bytes of FP8 experts")
+ if plan:
+ return out_dir
+
+ os.makedirs(out_dir, exist_ok=True)
+ index = resume_index(out_dir, src)
+ wanted = parse_layers(layers, g.layers)
+ todo = [i for i in wanted if i not in index["done"]]
+ if len(wanted) > len(todo):
+ print(f" {len(wanted) - len(todo)} layer(s) already converted, skipped")
+ write_index(out_dir, index)
+
+ pool = None
+ if workers > 1 and todo:
+ from multiprocessing import get_context
+ pool = get_context("spawn").Pool(workers, initializer=_init_worker, initargs=(model_dir,))
+ else:
+ _init_worker(model_dir)
+ worst = 0.0
+ try:
+ for layer in todo:
+ started = time.time()
+ errors, worker_mb = write_layer(src, out_dir, layer, pool)
+ index["done"] = sorted(index["done"] + [layer])
+ index["complete"] = len(index["done"]) == g.layers
+ write_index(out_dir, index)
+ worst = max(worst, float(errors.max()))
+ print(f" layer {layer:3d}: {g.experts} experts in {time.time() - started:6.1f}s "
+ f"rel L2 gate {errors[:, 0].mean():.4f} up {errors[:, 1].mean():.4f} "
+ f"down {errors[:, 2].mean():.4f} (max {errors.max():.4f}) "
+ f"peak RSS {peak_rss_mb():.0f} MB"
+ + (f" + {workers} workers x {worker_mb:.0f} MB" if pool else ""), flush=True)
+ finally:
+ if pool:
+ pool.close()
+ pool.join()
+ state = "complete" if index["complete"] else f"{len(index['done'])}/{g.layers} layers"
+ print(f"{out_dir}: {state}" + (f", worst expert matrix rel L2 {worst:.4f}" if todo else ""))
+ return out_dir
+
+
+def main():
+ ap = argparse.ArgumentParser(description="Qwen3.8 routed experts FP8 -> int4-g64 sidecar")
+ ap.add_argument("--model", required=True, help="the Qwen3.8 snapshot (config.json + shards)")
+ ap.add_argument("--out", help=f"sidecar directory (default: <model>/{SIDECAR_DIR}, where the engine looks)")
+ ap.add_argument("--workers", type=int, default=os.cpu_count() or 1,
+ help="processes converting experts in parallel (default: every CPU)")
+ ap.add_argument("--layers", help="only these layers, e.g. 0-3,7 (the index stays incomplete)")
+ ap.add_argument("--plan", action="store_true", help="print sizes and exit without writing")
+ a = ap.parse_args()
+ convert(a.model, a.out, a.workers, a.layers, a.plan)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/c/tools/make_qwen38_tiny.py b/c/tools/make_qwen38_tiny.py
index 6b34b9c5..7e53aeac 100644
--- a/c/tools/make_qwen38_tiny.py
+++ b/c/tools/make_qwen38_tiny.py
@@ -20,6 +20,7 @@ import argparse
import hashlib
import json
import random
+import shutil
import sys
from pathlib import Path
@@ -220,7 +221,45 @@ def _rewrite_shard_fp8(out: Path, packed):
print(f"fp8 experts: {len(packed)} matrices rewritten as F8_E4M3 + BF16 weight_scale_inv")
-def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_experts=False):
+def _int4_experts_in_model(model, out: Path):
+ """Write the int4-g64 sidecar of the fixture just saved, with the converter
+ the real model goes through (tools/convert_qwen38_experts_int4.py), then read its bytes
+ back and put the dequantized experts into the model in place, in float32:
+ every int4 value (code times an f32 scale) is exact there, and the other
+ BF16 weights widen exactly, so the reference computed afterwards is the
+ arithmetic of the sidecar with nothing rounded on the way. In BF16 the
+ reference's own rounding would be larger than the gap it has to see."""
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
+ import convert_qwen38_experts_int4 as int4
+ from safetensors.numpy import load_file
+ sidecar = Path(int4.convert(str(out), workers=1))
+ model.float()
+ stored = {}
+ for path in sorted(sidecar.glob("*.safetensors")):
+ stored.update(load_file(str(path)))
+ with torch.no_grad():
+ for name, param in model.named_parameters():
+ if name.endswith(".mlp.experts.gate_up_proj"):
+ base = name[: -len("gate_up_proj")]
+ E, twoI, H = param.shape; I = twoI // 2
+ parts = (("gate_proj", slice(0, I), H), ("up_proj", slice(I, twoI), H))
+ elif name.endswith(".mlp.experts.down_proj"):
+ base = name[: -len("down_proj")]
+ E, H, I = param.shape
+ parts = (("down_proj", slice(0, H), I),)
+ else:
+ continue
+ for e in range(E):
+ for kind, rows, cols in parts:
+ key = f"{base}{e}.{kind}.weight"
+ codes = int4.unpack_planar(stored[key], cols)
+ values = int4.dequantize(codes, stored[key + ".qs"])
+ param.data[e, rows, :] = torch.from_numpy(values)
+ print(f"int4 experts: {sidecar} written, the model now holds its dequantized values")
+
+
+def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_experts=False,
+ int4_experts=False, expert_gain=1.0):
if max_new < 1:
raise ValueError("max_new must be at least 1")
random.seed(seed)
@@ -292,8 +331,25 @@ def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_e
# save/load round trip (and follows the production arithmetic path).
model = model.to(dtype=torch.bfloat16)
model.eval()
+ if expert_gain != 1.0:
+ # At the default initialization the routed experts barely reach the
+ # logits: zeroing them all moves the final logits less than the
+ # oracle's tolerance (cosine 0.99994), so no end-to-end check can see
+ # their arithmetic. With a gain of 3 (applied before any quantization,
+ # so the scaled values simply are the fixture's experts) the int4
+ # fixture's FP8 and int4 references part after the third generated
+ # token, and the token gate itself tells one representation from the
+ # other. At 4 a routing near-tie flips between the C engine and the
+ # reference.
+ with torch.no_grad():
+ for name, param in model.named_parameters():
+ if ".mlp.experts." in name:
+ param.mul_(expert_gain)
packed = _fp8_experts_in_model(model) if fp8_experts else None
out.mkdir(parents=True, exist_ok=True)
+ # A sidecar left by an earlier run belongs to the weights it was converted
+ # from, and qwen38 picks it up by itself: regenerating the fixture drops it.
+ shutil.rmtree(out / "experts-int4g64", ignore_errors=True)
model.save_pretrained(str(out), safe_serialization=True)
if packed:
_rewrite_shard_fp8(out, packed)
@@ -310,6 +366,7 @@ def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_e
"transformers_version": transformers.__version__,
"text_only": True,
"fp8_experts": bool(fp8_experts),
+ **({"expert_gain": expert_gain} if expert_gain != 1.0 else {}),
"naming": "upstream Qwen4ExpForCausalLM (no model.language_model prefix)",
"config_summary": {
"hidden_size": config.hidden_size,
@@ -326,6 +383,15 @@ def build(out: Path, prompt_ids=None, max_new=8, seed=SEED, emit_ref=True, fp8_e
ref_path = out / "ref.json"
ref_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
print(f"Reference written to {ref_path}")
+ if int4_experts:
+ _int4_experts_in_model(model, out)
+ payload["int4_experts"] = True
+ payload["reference_dtype"] = "float32"
+ payload.update(_reference(model, prompt_ids, max_new))
+ if emit_ref:
+ ref_path = out / "ref_int4.json"
+ ref_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+ print(f"int4 reference written to {ref_path}")
print(f"Tiny Qwen3.8 fixture written to {out}")
print(f"prompt_ids={payload['prompt_ids']}")
print(f"full_ids={payload['full_ids']}")
@@ -342,10 +408,17 @@ def main():
help="routed experts as F8_E4M3 with 128x128 block weight_scale_inv "
"sidecars (the release layout); the reference uses the same "
"quantized values")
+ parser.add_argument("--expert-gain", type=float, default=1.0,
+ help="scale the routed experts so they visibly move the logits "
+ "(the int4 fixture uses 3)")
+ parser.add_argument("--int4-experts", action="store_true",
+ help="also convert the routed experts to the experts-int4g64/ sidecar "
+ "and write ref_int4.json, the reference of the dequantized sidecar")
args = parser.parse_args()
prompt = [int(x) for x in args.prompt_ids.split(",") if x.strip()] if args.prompt_ids else None
build(args.out, prompt_ids=prompt, max_new=args.max_new, seed=args.seed, emit_ref=not args.no_ref,
- fp8_experts=args.fp8_experts)
+ fp8_experts=args.fp8_experts, int4_experts=args.int4_experts,
+ expert_gain=args.expert_gain)
if __name__ == "__main__":