Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(qwen38): use native weights and improve metrics
themorlock2d65953208b3
diff --git a/README.md b/README.mdindex 8b1f1d52..8e1f5cd8 100644--- a/README.md+++ b/README.md@@ -408,7 +408,7 @@ the model's `config.json`):> | **Inkling** | ~469 GB | 25 GB with the int4 dense container, ~120 GB without | not needed |> | **Kimi K3** | ~1.6 TB | 32 GB+ | not needed |> | **DeepSeek V4 Flash** | ~167 GB | 16 GB min, 32 GB comfortable | optional; any NVIDIA card from the GTX 10 series up (Pascal/Turing via `CUDA_ARCH=portable-pre-ampere NO_TC=1`, best on RTX 50) makes prefill 5-10x and decode ~2.5x faster |-> | **Qwen3.8-Flash-Next** | ~185.5 GB (official FP8 checkpoint) | 32 GB recommended at the default context | not supported; CPU only |+> | **Qwen3.8-Flash-Next** | ~185.5 GB (official FP8 checkpoint) | 16 GB min, 24 GB comfortable at the default context | not supported; CPU only |> | **Qwen3.6-35B-A3B** | ~20 GB (int4-gs64 container) | 24 GB (needs full RAM residency) | optional; the CUDA VRAM expert tier measured **1.44 -> 10.05 tok/s (7.0x)** on two 8 GB cards, output bit-identical to CPU |>> A GPU only ever makes it faster. Speed is set by your disk, because the expertsdiff --git a/c/Makefile b/c/Makefileindex 66c36114..c0b03e1f 100644--- a/c/Makefile+++ b/c/Makefile@@ -1014,8 +1014,9 @@ qwen38-tiny-generate:$(PYTHON) tools/make_qwen38_tiny.py --out ./qwen38_tinyqwen38-tiny-check: qwen38-tiny-generate qwen38$(EXE)- @for cap in 1 4; do \- OMP_NUM_THREADS=2 SNAP=./qwen38_tiny ./qwen38$(EXE) $$cap 8 ./qwen38_tiny/ref.json || exit $$?; \+ @for bf16 in 0 1; do for cap in 1 4; do \+ Q38_NATIVE_BF16=$$bf16 OMP_NUM_THREADS=2 SNAP=./qwen38_tiny ./qwen38$(EXE) $$cap 8 ./qwen38_tiny/ref.json || exit $$?; \+ done; \done# Context-size gates: KV layout, growth across requests, and the attention@@ -1111,6 +1112,12 @@ tests/test_qwen38_config$(EXE): tests/test_qwen38_config.c qwen38.c qwen38_core.tests/test_qwen38_serve_framing$(EXE): tests/test_qwen38_serve_framing.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h route_trace.h serve_codec.h tok_unicode.h tok_unicode_o200k.h$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)+tests/test_qwen38_metrics$(EXE): tests/test_qwen38_metrics.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h route_trace.h serve_codec.h tok_unicode.h tok_unicode_o200k.h+ $(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)++tests/test_qwen38_native_weights$(EXE): tests/test_qwen38_native_weights.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h segment_runtime.c segment_runtime.h segment_adapters.h segment_adapter_internal.h st.h json.h compat.h quant.h route_trace.h serve_codec.h tok_unicode.h tok_unicode_o200k.h+ $(CC) $(NOCUDA_CFLAGS) $< segment_runtime.c -o $@ $(NOCUDA_LDFLAGS)+tests/test_st_map$(EXE): tests/test_st_map.c st.h json.h compat.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/coli b/c/coliindex ba1f5797..13cef1cf 100755--- a/c/coli+++ b/c/coli@@ -523,18 +523,21 @@ def env_for_engine(a, arch, plan=None):# code path, so #1196 could measure and save a perfectly valid Kimi,# Inkling, OLMoE, Qwen or V4 profile that chat/serve never loaded.from resource_plan import build_plan, environment_for_plan- if plan is None:- ram,ctx,devices,vram=resource_request(a,env)- plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy,- kv_slots=requested_kv_slots(a,env))- # Preserve the existing non-GLM GPU contract: explicit --gpu/--vram- # selects a capable build above. Merely asking for auto-tier must not- # turn a CPU-only sibling binary into an attempted CUDA launch.- plan_must_not_choose_omp=("OMP_NUM_THREADS" not in env- and (arch == "deepseek_v4"- or env.get("COLI_NO_OMP_TUNE")))- env=environment_for_plan(plan,env,- cuda_enabled=env.get("COLI_CUDA") == "1")+ try:+ if plan is None:+ ram,ctx,devices,vram=resource_request(a,env)+ plan=build_plan(a.model,ram,ctx,devices,vram,policy=a.policy,+ kv_slots=requested_kv_slots(a,env))+ # Preserve the existing non-GLM GPU contract: explicit --gpu/--vram+ # selects a capable build above. Merely asking for auto-tier must not+ # turn a CPU-only sibling binary into an attempted CUDA launch.+ plan_must_not_choose_omp=("OMP_NUM_THREADS" not in env+ and (arch == "deepseek_v4"+ or env.get("COLI_NO_OMP_TUNE")))+ env=environment_for_plan(plan,env,+ cuda_enabled=env.get("COLI_CUDA") == "1")+ except (OSError,ValueError,json.JSONDecodeError) as error:+ sys.exit(f"{C.yel}invalid resource plan:{C.r} {error}")if plan_must_not_choose_omp:# environment_for_plan supplies the physical-core default used by# the other engines. V4 deliberately computes logical CPUs minusdiff --git a/c/family_registry.py b/c/family_registry.pyindex 811e0234..dd0b004c 100644--- a/c/family_registry.py+++ b/c/family_registry.py@@ -532,7 +532,7 @@ _QWEN38_EXPERT = re.compile()_QWEN38_EXPERT_SCALE = re.compile(r"^(?:model\.)?(?:language_model\.)?layers\.(\d+)\.mlp\.experts\."- r"(\d+)\.(?:gate_proj|up_proj|down_proj)\.weight_scale_inv$"+ r"(\d+)\.(gate_proj|up_proj|down_proj)\.weight_scale_inv$")_QWEN38_FUSED_EXPERT = re.compile(r"^(?:model\.)?(?:language_model\.)?layers\.(\d+)\.mlp\.experts\."@@ -541,7 +541,7 @@ _QWEN38_FUSED_EXPERT = re.compile(def _individual_expert_inventory(pattern):- def inventory(name, size, _config):+ def inventory(name, size, _config, _dtype=None):match = pattern.search(name)if match is None:return ()@@ -549,7 +549,7 @@ def _individual_expert_inventory(pattern):return inventory-def _inkling_expert_inventory(name, size, config):+def _inkling_expert_inventory(name, size, config, _dtype=None):match = _INKLING_EXPERT.fullmatch(name)if match is None:return ()@@ -562,7 +562,7 @@ def _inkling_expert_inventory(name, size, config):return tuple((layer, expert, per_expert) for expert in range(experts))-def _qwen38_expert_inventory(name, size, config):+def _qwen38_expert_inventory(name, size, config, dtype=None):"""Index both per-expert FP8 and fused BF16 expert tensor layouts.Resource planning calls this once per tensor; summing the returned byte@@ -576,12 +576,42 @@ def _qwen38_expert_inventory(name, size, config):elements = hidden * intermediateif size not in (elements, elements * 2, elements * 4):raise ValueError(f"qwen38: expert tensor {name!r} has unexpected size {size}")- # All supported source dtypes are decoded to f32 in q38_load_expert().- return ((int(match.group(1)), int(match.group(2)), elements * 4),)+ inferred = {elements: "F8_E4M3", elements * 2: "BF16",+ elements * 4: "F32"}[size]+ dtype = inferred if dtype is None else dtype+ expected = {"F8_E4M3": elements, "F8_E4M3FN": elements,+ "float8_e4m3fn": elements, "BF16": elements * 2,+ "F16": elements * 2, "F32": elements * 4}.get(dtype)+ if expected != size:+ raise ValueError(f"qwen38: expert tensor {name!r} has unsupported "+ f"dtype/size {dtype}/{size}")+ # Native FP8/BF16 retain their source representation. F16 is accepted+ # by the engine but expands to F32, so its two-byte file size is not a+ # safe cache budget.+ retained = size if dtype not in ("F16",) else elements * 4+ return ((int(match.group(1)), int(match.group(2)), retained),)match = _QWEN38_EXPERT_SCALE.fullmatch(name)if match:- # The scale sidecar is streamed with its expert, never dense-resident.- return ((int(match.group(1)), int(match.group(2)), 0),)+ # Slots normalize every supported source scale dtype to F32. Derive+ # the retained grid rather than charging the source byte count (the+ # official sidecars happen to be BF16).+ hidden = _required_int(config, "hidden_size", "qwen38")+ intermediate = _required_int(config, "moe_intermediate_size", "qwen38")+ projection = match.group(3)+ rows = intermediate if projection != "down_proj" else hidden+ cols = hidden if projection != "down_proj" else intermediate+ scale_count = math.ceil(rows / 128) * math.ceil(cols / 128)+ if dtype is None:+ if size == scale_count * 4:+ dtype = "F32"+ elif size == scale_count * 2:+ dtype = "BF16"+ source_bytes = {"BF16": 2, "F16": 2, "F32": 4}.get(dtype)+ if source_bytes is None or size != scale_count * source_bytes:+ raise ValueError(f"qwen38: expert scale {name!r} has unsupported "+ f"dtype/size {dtype}/{size}")+ scale_bytes = scale_count * 4+ return ((int(match.group(1)), int(match.group(2)), scale_bytes),)match = _QWEN38_FUSED_EXPERT.fullmatch(name)if not match:return ()@@ -592,13 +622,49 @@ def _qwen38_expert_inventory(name, size, config):elements = experts * matrices * hidden * intermediateif size not in (elements * 2, elements * 4):raise ValueError(f"qwen38: fused expert tensor {name!r} has unexpected size {size}")- per_expert = matrices * hidden * intermediate * 4+ inferred = "BF16" if size == elements * 2 else "F32"+ dtype = inferred if dtype is None else dtype+ expected = {"BF16": elements * 2, "F16": elements * 2,+ "F32": elements * 4}.get(dtype)+ if expected != size:+ raise ValueError(f"qwen38: fused expert tensor {name!r} has unsupported "+ f"dtype/size {dtype}/{size}")+ per_expert = (elements * 4 if dtype == "F16" else size) // expertslayer = int(match.group(1))return tuple((layer, expert, per_expert) for expert in range(experts))-def _qwen38_resident_inventory(name, size, _config):- """Resident f32 bytes for tensors the native text engine actually loads."""+_QWEN38_NATIVE_MATRIX_SUFFIXES = (+ "embed_tokens.weight",+ "input_mix_weight_down.weight",+ "input_mix_weight_up.weight",+ "block_inject_weight.weight",+ "mlp.gate.weight",+ "mlp.shared_expert.gate_proj.weight",+ "mlp.shared_expert.up_proj.weight",+ "mlp.shared_expert.down_proj.weight",+ "self_attn.q_proj.weight",+ "self_attn.k_proj.weight",+ "self_attn.v_proj.weight",+ "self_attn.o_proj.weight",+ "self_attn.indexer.index_qk_proj.weight",+ "linear_attn.in_proj_qkv.weight",+ "linear_attn.in_proj_z.weight",+ "linear_attn.in_proj_b.weight",+ "linear_attn.in_proj_a.weight",+ "linear_attn.out_proj.weight",+ "ple.key_proj.weight",+ "ple.value_proj.weight",+)+++def _qwen38_resident_inventory(name, size, _config, dtype=None):+ """Resident bytes for tensors the native text engine actually loads.++ The official checkpoints store non-quantized text tensors as BF16. Major+ rank-two matrices stay in that two-byte representation; the comparatively+ small norms, gates and convolution vectors still expand through st_read_f32.+ """if name.startswith("mtp.") or name.startswith("model.visual."):return 0if ".ple.ple_embedding.ngram_embedding." in name and name.endswith(".weight"):@@ -614,7 +680,21 @@ def _qwen38_resident_inventory(name, size, _config):name.startswith("model.language_model.") or(name.startswith("model.") andnot name.startswith("model.visual.")))- return size * 2 if text_tensor else 0 # supported checkpoints store BF16.+ if not text_tensor:+ return 0+ # Direct registry tests predate dtype-bearing planner entries and use BF16+ # sizes; real scans always supply the safetensors dtype.+ dtype = "BF16" if dtype is None else dtype+ element_bytes = {"BF16": 2, "F16": 2, "F32": 4}.get(dtype)+ if not element_bytes:+ raise ValueError(f"qwen38: resident tensor {name!r} has unsupported dtype {dtype}")+ elements = size // element_bytes+ if elements * element_bytes != size:+ raise ValueError(f"qwen38: resident tensor {name!r} has invalid byte size {size}")+ if (name == "lm_head.weight" or+ name.endswith(_QWEN38_NATIVE_MATRIX_SUFFIXES)) and dtype == "BF16":+ return size+ return elements * 4COMMON_CAP = FamilyCapabilities(False, False, False, True)@@ -935,23 +1015,27 @@ def planner_geometry(resolved, context):return geometry-def expert_contributions(resolved, name, size):+def expert_contributions(resolved, name, size, dtype=None):if isinstance(size, bool) or not isinstance(size, int) or size < 0:raise ValueError("tensor size must be a non-negative integer")+ if dtype is not None and not isinstance(dtype, str):+ raise ValueError("tensor dtype must be a string")contributions = resolved.descriptor.expert_inventory(- name, size, resolved.family_config)+ name, size, resolved.family_config, dtype)for layer, expert, byte_count in contributions:if layer < 0 or expert < 0 or byte_count < 0:raise RegistryError(f"invalid expert inventory for {resolved.descriptor.id}")return contributions-def resident_contribution(resolved, name, size):+def resident_contribution(resolved, name, size, dtype=None):if isinstance(size, bool) or not isinstance(size, int) or size < 0:raise ValueError("tensor size must be a non-negative integer")inventory = resolved.descriptor.resident_inventory+ if dtype is not None and not isinstance(dtype, str):+ raise ValueError("tensor dtype must be a string")contribution = size if inventory is None else inventory(- name, size, resolved.family_config)+ name, size, resolved.family_config, dtype)if isinstance(contribution, bool) or not isinstance(contribution, int) or contribution < 0:raise RegistryError(f"invalid resident inventory for {resolved.descriptor.id}")return contributiondiff --git a/c/qwen38.c b/c/qwen38.cindex a4122446..7cf950d2 100644--- a/c/qwen38.c+++ b/c/qwen38.c@@ -7,7 +7,7 @@** Resident tensors are loaded from the official multimodal checkpoint's text* namespace. Vision and MTP tensors are indexed but never read. Official- * block-FP8 experts are decoded into a bounded per-layer LRU; the 51B-parameter+ * block-FP8 experts remain native in a bounded per-layer LRU; the 51B-parameter* PLE table remains on disk and only its sixteen 160-byte rows/token are read.** Runtime: SNAP=<dir>, optional Q38_MAXT, and argv@@ -558,6 +558,7 @@ cleanup:if(result)free_tokenizer();return result;}+/* Decode token ids to text using g_tok, writing to stdout. Handles Qwen's* byte-representation markers (Ġ=space, Ċ=newline, ▁=space) and <0xXX> byte* fallback. Only active when a tokenizer was loaded. */@@ -579,6 +580,8 @@ static int q38_completion_nonce(double monotonic_seconds){double within_ten=fmod(monotonic_seconds,10.0);return (int)(within_ten*1000.0)%10000;}+/* Decode throughput is reciprocal TPOT: prefill produces token one, so N+ * completion tokens contain exactly N-1 measured decode intervals. */static double q38_decode_rate(int generated,double decode_seconds){return generated>1&&isfinite(decode_seconds)&&decode_seconds>1e-6?(double)(generated-1)/decode_seconds:0.0;@@ -1196,6 +1199,29 @@ static int serve_eos_ids(int *ids,int cap,int config_eos,int vocab){return n;}+static double q38_cache_hit_percent(uint64_t hits,uint64_t misses){+ double total=(double)hits+(double)misses;+ return total?100.0*(double)hits/total:0.0;+}++static int q38_format_prof(char *out,size_t capacity,double wall_s,int prompt_tokens,+ int completion_tokens,const Q38Timers *timers){+ double expert_read=timers->seconds[Q38_TM_EXPERT_READ];+ double fp8_expand=timers->seconds[Q38_TM_FP8_EXPAND];+ double expert_compute=timers->seconds[Q38_TM_ROUTED_EXPERT]++ timers->seconds[Q38_TM_SHARED_EXPERT];+ double attention=timers->seconds[Q38_TM_DELTANET]++ timers->seconds[Q38_TM_QSA_INDEX]++ timers->seconds[Q38_TM_QSA_ATTENTION];+ int count=snprintf(out,capacity,+ "PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n",+ wall_s,prompt_tokens,completion_tokens,expert_read,+ expert_read+fp8_expand,expert_compute,attention,+ timers->seconds[Q38_TM_LM_HEAD],+ (unsigned long long)timers->forwards);+ return count>=0&&(size_t)count<capacity?count:-1;+}+static int serve_one(Model *m, ServeReq *q){int *ids=NULL, np=0;encode_text_n(q->payload,(size_t)q->plen,&ids,&np); /* byte-counted prompt; qwen38 adds no BOS */@@ -1206,12 +1232,15 @@ static int serve_one(Model *m, ServeReq *q){fflush(stdout); free(ids); return 0;}printf("ACCEPT %s %d\n",q->id,np); fflush(stdout);+ double request_started=now_s();+ uint64_t hits_before=m->hits, misses_before=m->miss;+ Q38Timers timers_before=m->timers;m->max_t = np + q->max_tok;reset_recurrent(m); ensure_kv(m); m->kv_len = 0;float *lo = step(m, ids, np, 0);int gen=0, limited=1, cancelled=0, stopped=0, input_eof=0;int eos_ids[4];int n_eos=serve_eos_ids(eos_ids,4,m->c.eos_id,m->c.vocab);- double decode_t0=0.0;+ double first_token_at=0.0,last_token_at=0.0;unsigned char sbuf[16]; int sbn=0;for(int s=0;s<q->max_tok;s++){while(!input_eof&&coli_stdin_readable()){@@ -1225,7 +1254,9 @@ static int serve_one(Model *m, ServeReq *q){free(lo); lo=NULL;int is_eos=0; for(int e=0;e<n_eos;e++) if(tk==eos_ids[e]) is_eos=1;if(is_eos){ limited=0; break; }- if(!gen)decode_t0=now_s();+ double token_at=now_s();+ if(!gen)first_token_at=token_at;+ last_token_at=token_at;unsigned char *token=NULL;int token_n=0;if(decode_id_alloc(tk,&token,&token_n)){fprintf(stderr,"[decode] out of memory\n");exit(1);}if(token_n>(INT_MAX-3)/3){free(token);fprintf(stderr,"[decode] token is too large\n");exit(1);}@@ -1258,10 +1289,21 @@ static int serve_one(Model *m, ServeReq *q){fprintf(stderr,"[decode] invalid UTF-8 finalization\n");exit(1);}if(tail_n)serve_data(q->id,(char*)tail,tail_n);- double dt=gen>1?now_s()-decode_t0:0.0;- printf("DONE %s STAT %d %.3f %.1f %.2f %d %d\n",q->id,gen,- q38_decode_rate(gen,dt),0.0,rss_gb(),np,limited);+ double wall_s=now_s()-request_started;+ double decode_s=gen>1?last_token_at-first_token_at:0.0;+ uint64_t hits=m->hits-hits_before,misses=m->miss-misses_before;+ Q38Timers timers=q38_tm_delta(&m->timers,&timers_before);+ ColiServeDone done={gen,q38_decode_rate(gen,decode_s),+ q38_cache_hit_percent(hits,misses),rss_gb(),np,limited};+ coli_serve_write_done(stdout,q->id,&done);+ /* Stable dashboard contract. Disk service and synchronous miss wait+ * intentionally overlap; all finer Qwen phases remain available through+ * COLI_TIMERS=1 without widening the shared frame. */+ char profile[256];int profile_bytes=q38_format_prof(profile,sizeof profile,wall_s,np,gen,&timers);+ if(profile_bytes>0)fwrite(profile,1,(size_t)profile_bytes,stdout);+ else fprintf(stderr,"[qwen38] internal error: PROF frame overflow\n");fflush(stdout);+ q38_tm_report_bank(&timers,"request");return input_eof?-1:0;}@@ -1333,6 +1375,7 @@ int main(int argc, char **argv) {"[enc] a complete tokenizer.json is required (put it in SNAP or set TOK)\n");free_tokenizer();return 1;}+if (serve_mode) {/* no argv prompt to load */} else if (is_ref) {@@ -1489,7 +1532,7 @@ int main(int argc, char **argv) {}double tot = m.hits + m.miss;if (g_ttft >= 0) fprintf(stderr, "TTFT: %.2f s (time to first token)\n", g_ttft);- tm_report();+ tm_report(&m);fprintf(stderr, "\nPEAK RSS: %.2f GB\n", rss_gb());fprintf(stderr, "Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,(unsigned long long)m.hits, (unsigned long long)m.miss);@@ -1515,21 +1558,169 @@ static int q38_u64_add(uint64_t *total,uint64_t value){if(!total||*total>UINT64_MAX-value)return -1;*total+=value;return 0;}-static int q38_segment_cache_capacity(const Cfg *c,uint32_t range_layers,- uint64_t memory_limit,int *capacity,- uint64_t *slot_bytes){- uint64_t cells,bytes;- if(!c||!range_layers||!capacity||q38_u64_mul((uint64_t)c->hidden,- (uint64_t)c->inter,&cells)||q38_u64_mul(cells,3u,&cells)||- q38_u64_mul(cells,sizeof(float),&bytes))return -1;- if(slot_bytes)*slot_bytes=bytes;+enum {+ Q38_EXPERT_FP8_BLOCK = 1u << 0,+ Q38_EXPERT_FP8_EXPANDED = 1u << 1,+ Q38_EXPERT_BF16 = 1u << 2,+ Q38_EXPERT_F16 = 1u << 3,+ Q38_EXPERT_F32 = 1u << 4+};++/* Return the bytes retained by one cache matrix after loading. The source+ * dtype alone is not enough: native FP8 retains its F32 block scales, native+ * BF16 retains two-byte elements, and both opt-out modes retain expanded F32.+ * Keeping this calculation beside the loader's representation choices makes+ * Segment's memory contract describe the allocation it actually performs. */+static int q38_segment_matrix_bytes(Model *m,const char *weight_name,+ int rows,int cols,uint64_t *bytes,+ unsigned *numeric_kinds){+ st_tensor *weight=st_find(&m->S,weight_name);uint64_t elements;+ if(!weight||weight->rank!=2||weight->shape[0]!=rows||+ weight->shape[1]!=cols||weight->numel!=(int64_t)rows*cols||+ q38_u64_mul((uint64_t)rows,(uint64_t)cols,&elements))return -1;+ if(weight->dtype==4){+ char scale_name[340];+ int length=snprintf(scale_name,sizeof scale_name,"%s_scale_inv",weight_name);+ int64_t block_rows=fp8_nblk(rows),block_cols=fp8_nblk(cols);+ st_tensor *scale=length>0&&(size_t)length<sizeof scale_name?+ st_find(&m->S,scale_name):NULL;+ if(!scale||scale->dtype<0||scale->dtype>2||scale->rank!=2||+ scale->shape[0]!=block_rows||scale->shape[1]!=block_cols||+ scale->numel!=block_rows*block_cols)return -1;+ if(m->native_fp8){+ uint64_t scale_bytes;+ if(q38_u64_mul((uint64_t)scale->numel,sizeof(float),&scale_bytes)||+ q38_u64_add(&elements,scale_bytes))return -1;+ *bytes=elements;*numeric_kinds|=Q38_EXPERT_FP8_BLOCK;+ }else{+ if(q38_u64_mul(elements,sizeof(float),bytes))return -1;+ *numeric_kinds|=Q38_EXPERT_FP8_EXPANDED;+ }+ return 0;+ }+ if(weight->dtype<0||weight->dtype>2)return -1;+ uint64_t element_bytes=weight->dtype==0&&m->native_bf16?+ sizeof(uint16_t):sizeof(float);+ if(q38_u64_mul(elements,element_bytes,bytes))return -1;+ *numeric_kinds|=weight->dtype==0?Q38_EXPERT_BF16:+ weight->dtype==1?Q38_EXPERT_F16:Q38_EXPERT_F32;+ return 0;+}++static int q38_segment_fused_matrix_bytes(Model *m,const st_tensor *tensor,+ int64_t elements,uint64_t *bytes,+ unsigned *numeric_kinds){+ if(!tensor||tensor->dtype<0||tensor->dtype>2||elements<1)return -1;+ uint64_t element_bytes=tensor->dtype==0&&m->native_bf16?+ sizeof(uint16_t):sizeof(float);+ if(q38_u64_mul((uint64_t)elements,element_bytes,bytes))return -1;+ *numeric_kinds|=tensor->dtype==0?Q38_EXPERT_BF16:+ tensor->dtype==1?Q38_EXPERT_F16:Q38_EXPERT_F32;+ return 0;+}++/* Sum the maximum possible resident expert slot for every layer in the loaded+ * range. Individual-expert containers are allowed to be heterogeneous, so a+ * layer is priced at its largest expert rather than assuming expert zero is+ * representative. Fused containers have one dtype/geometry for every expert. */+static int q38_segment_expert_layout(Model *m,uint32_t begin,uint32_t end,+ uint64_t *bytes_per_capacity,+ unsigned *numeric_kinds){+ if(!m||!bytes_per_capacity||!numeric_kinds||begin>=end||+ end>(uint32_t)m->c.layers)return -1;+ Cfg *c=&m->c;uint64_t range_bytes=0;unsigned kinds=0;+ for(uint32_t layer=begin;layer<end;layer++){+ char name[320];q38_name(m,name,sizeof name,(int)layer,+ "mlp.experts.gate_up_proj");+ st_tensor *gate_up=st_find(&m->S,name);uint64_t layer_bytes=0;+ if(gate_up){+ if(gate_up->rank!=3||gate_up->shape[0]!=c->experts||+ gate_up->shape[1]!=(int64_t)2*c->inter||+ gate_up->shape[2]!=c->hidden)return -1;+ uint64_t part;+ if(q38_segment_fused_matrix_bytes(m,gate_up,+ (int64_t)2*c->inter*c->hidden,&part,&kinds)||+ q38_u64_add(&layer_bytes,part))return -1;+ q38_name(m,name,sizeof name,(int)layer,"mlp.experts.down_proj");+ st_tensor *down=st_find(&m->S,name);+ if(!down||down->rank!=3||down->shape[0]!=c->experts||+ down->shape[1]!=c->hidden||down->shape[2]!=c->inter||+ q38_segment_fused_matrix_bytes(m,down,+ (int64_t)c->hidden*c->inter,&part,&kinds)||+ q38_u64_add(&layer_bytes,part))return -1;+ }else{+ const char *projection[3]={"gate_proj","up_proj","down_proj"};+ int rows[3]={c->inter,c->inter,c->hidden};+ int cols[3]={c->hidden,c->hidden,c->inter};+ for(int expert=0;expert<c->experts;expert++){+ uint64_t expert_bytes=0;+ for(int projection_index=0;projection_index<3;projection_index++){+ char suffix[192];+ int length=snprintf(suffix,sizeof suffix,+ "mlp.experts.%d.%s.weight",expert,+ projection[projection_index]);+ if(length<0||(size_t)length>=sizeof suffix)return -1;+ q38_name(m,name,sizeof name,(int)layer,suffix);+ uint64_t part;+ if(q38_segment_matrix_bytes(m,name,rows[projection_index],+ cols[projection_index],&part,&kinds)||+ q38_u64_add(&expert_bytes,part))return -1;+ }+ if(expert_bytes>layer_bytes)layer_bytes=expert_bytes;+ }+ }+ if(!layer_bytes||q38_u64_add(&range_bytes,layer_bytes))return -1;+ }+ *bytes_per_capacity=range_bytes;*numeric_kinds=kinds;return 0;+}++static int q38_segment_cache_capacity(uint64_t bytes_per_capacity,int experts,+ uint64_t memory_limit,int *capacity){+ if(!bytes_per_capacity||experts<1||!capacity)return -1;if(!memory_limit){*capacity=1;return 0;}- uint64_t slots=memory_limit/bytes/range_layers;+ uint64_t slots=memory_limit/bytes_per_capacity;if(!slots)return -1;- if(slots>(uint64_t)c->experts)slots=(uint64_t)c->experts;+ if(slots>(uint64_t)experts)slots=(uint64_t)experts;*capacity=(int)slots;return 0;}+static void q38_segment_numeric_class(char *out,size_t capacity,+ unsigned numeric_kinds){+ const char *known=NULL;+ switch(numeric_kinds){+ case Q38_EXPERT_FP8_BLOCK:+ known="qwen38/fp8-block-f32dot-f64blocksum/cpu-v1";break;+ case Q38_EXPERT_FP8_EXPANDED:+ known="qwen38/fp8-expanded-f32dot/cpu-v1";break;+ case Q38_EXPERT_BF16:+ known="qwen38/bf16-values-f32dot/cpu-v1";break;+ case Q38_EXPERT_F16:+ known="qwen38/f16-expanded-f32dot/cpu-v1";break;+ case Q38_EXPERT_F32:+ known="qwen38/f32dot/cpu-v1";break;+ default:+ /* The bitset is part of this opaque compatibility identifier. In+ * particular, FP8_BLOCK and FP8_EXPANDED are different bits, so a+ * heterogeneous range cannot silently change its reduction rules+ * while continuing to claim snapshot/numeric compatibility. */+ snprintf(out,capacity,"qwen38/mixed-expert-arithmetic-%02x/cpu-v1",+ numeric_kinds);return;+ }+ snprintf(out,capacity,"%s",known);+}++static int q38_segment_cache_resize(Model *m,int capacity){+ if(!m||capacity<1)return -1;+ for(int layer=m->range_begin;layer<m->range_end;layer++){+ LCache *cache=&m->cache[layer];+ if(cache->n)return -1;+ Slot *slots=(Slot*)calloc((size_t)capacity,sizeof(*slots));+ if(!slots)return -1;+ free(cache->slots);cache->slots=slots;cache->cap=capacity;+ }+ return 0;+}+static int q38_segment_session_state_bytes(const Cfg *c,uint32_t begin,uint32_t end,uint32_t context,uint64_t *out){@@ -1607,19 +1798,35 @@ static int qwen38_segment_engine_open(if (!options->context_tokens || options->context_tokens>(uint32_t)maxctx || options->context_tokens>INT_MAX){free(probe.is_attn);return coli_segment_adapter_error(error,error_size,"Qwen3.8 Segment context exceeds model limit");}- int cap=1;uint64_t slot_bytes=0;- if(q38_segment_cache_capacity(&probe,end-options->layer_begin,- options->memory_limit_bytes,&cap,&slot_bytes)){- free(probe.is_attn);- return coli_segment_adapter_error(error,error_size,- "Qwen3.8 Segment memory limit cannot hold one expert per loaded layer");- }free(probe.is_attn);Qwen38SegmentEngine *e=(Qwen38SegmentEngine*)calloc(1,sizeof(*e));if(!e) return coli_segment_adapter_error(error,error_size,"out of memory opening Qwen3.8 Segment");e->layer_begin=options->layer_begin; e->layer_end=end; e->context_tokens=options->context_tokens;if(pthread_mutex_init(&e->run_lock,NULL)){free(e);return coli_segment_adapter_error(error,error_size,"cannot initialize Qwen3.8 Segment lock");}- model_init_range(&e->model,options->model_dir,cap,8,(int)e->layer_begin,(int)e->layer_end,0,0);+ /* Open with a single empty cache slot so tensor metadata and the effective+ * native-storage switches are available before honoring the caller's+ * budget. No expert is loaded during model_init_range, so resizing here+ * cannot discard data or perturb LRU state. */+ model_init_range(&e->model,options->model_dir,1,8,+ (int)e->layer_begin,(int)e->layer_end,0,0);+ uint64_t bytes_per_capacity=0;unsigned numeric_kinds=0;int cap=1;+ if(q38_segment_expert_layout(&e->model,e->layer_begin,e->layer_end,+ &bytes_per_capacity,&numeric_kinds)){+ q38_model_free(&e->model);pthread_mutex_destroy(&e->run_lock);free(e);+ return coli_segment_adapter_error(error,error_size,+ "Qwen3.8 Segment checkpoint has an unsupported expert layout");+ }+ if(q38_segment_cache_capacity(bytes_per_capacity,e->model.c.experts,+ options->memory_limit_bytes,&cap)){+ q38_model_free(&e->model);pthread_mutex_destroy(&e->run_lock);free(e);+ return coli_segment_adapter_error(error,error_size,+ "Qwen3.8 Segment memory limit cannot hold one expert per loaded layer");+ }+ if(cap!=1&&q38_segment_cache_resize(&e->model,cap)){+ q38_model_free(&e->model);pthread_mutex_destroy(&e->run_lock);free(e);+ return coli_segment_adapter_error(error,error_size,+ "out of memory sizing Qwen3.8 Segment expert cache");+ }e->model.max_t=(int)e->context_tokens; e->model.kv_cap=(int)e->context_tokens;memset(capabilities,0,sizeof(*capabilities)); capabilities->struct_size=sizeof(*capabilities);capabilities->abi_version=COLI_SEGMENT_ABI_VERSION;@@ -1627,7 +1834,8 @@ static int qwen38_segment_engine_open(COLI_SEGMENT_CAP_MULTI_SESSION|COLI_SEGMENT_CAP_CPU;coli_segment_capability_string(capabilities->engine_id,sizeof(capabilities->engine_id),"qwen38");coli_segment_capability_string(capabilities->state_schema,sizeof(capabilities->state_schema),"qwen38/hyper-kv-deltanet-ple-f32-v1");- snprintf(capabilities->numeric_class,sizeof(capabilities->numeric_class),"qwen38/f32-fp8experts/cpu-v1");+ q38_segment_numeric_class(capabilities->numeric_class,+ sizeof(capabilities->numeric_class),numeric_kinds);capabilities->state_dtype=COLI_SEGMENT_DTYPE_F32; capabilities->state_width=(uint32_t)e->model.c.hc_width;capabilities->max_batch_rows=128; capabilities->max_context_tokens=(uint32_t)maxctx; capabilities->num_layers=(uint32_t)layers;*engine_impl=e; return 0;diff --git a/c/qwen38_core.h b/c/qwen38_core.hindex cf710611..e6bf42f1 100644--- a/c/qwen38_core.h+++ b/c/qwen38_core.h@@ -27,27 +27,71 @@ typedef struct {uint8_t *is_attn;} Cfg;-typedef struct { float *norm, *down, *up, *inject; } GatedResidual;+typedef enum {+ Q38_WEIGHT_NONE = 0,+ Q38_WEIGHT_F32,+ Q38_WEIGHT_BF16,+ Q38_WEIGHT_FP8+} Q38WeightKind;++typedef struct {+ void *data;+ float *scales; /* block-FP8 only */+ int rows, cols;+ int64_t elements, scale_count;+ Q38WeightKind kind;+} Q38Weight;++typedef struct { float *norm; Q38Weight down, up, inject; } GatedResidual;++/* Persistent wall-clock counters. They live on Model rather than in process+ * globals so Segment sessions and future multi-model serving cannot leak phase+ * time into one another. Some categories intentionally overlap: architecture+ * phases (DeltaNet/QSA/PLE) contain resident matmuls, while the matmul counter+ * answers the orthogonal question "how much time is in dense kernels?". */+typedef enum {+ Q38_TM_EXPERT_READ = 0,+ Q38_TM_FP8_EXPAND,+ Q38_TM_ROUTED_EXPERT,+ Q38_TM_SHARED_EXPERT,+ Q38_TM_DENSE_MATMUL,+ Q38_TM_DELTANET,+ Q38_TM_QSA_INDEX,+ Q38_TM_QSA_ATTENTION,+ Q38_TM_PLE,+ Q38_TM_LM_HEAD,+ Q38_TM_COUNT+} Q38Timer;++typedef struct {+ double seconds[Q38_TM_COUNT];+ uint64_t forwards;+} Q38Timers;typedef struct {GatedResidual attn_gr, mlp_gr;- float *router, *sh_g, *sh_u, *sh_d, *sh_gate;- float *q, *k, *v, *o, *qn, *kn;- float *idx_qk, *idx_qn, *idx_kn;- float *dn_qkv, *dn_z, *dn_b, *dn_a, *dn_conv;- float *dn_dtbias, *dn_alog, *dn_norm, *dn_out;- float *ple_key, *ple_value, *ple_norm_key, *ple_norm_query;+ Q38Weight router, sh_g, sh_u, sh_d;+ float *sh_gate;+ Q38Weight q, k, v, o;+ float *qn, *kn;+ Q38Weight idx_qk;+ float *idx_qn, *idx_kn;+ Q38Weight dn_qkv, dn_z, dn_b, dn_a, dn_out;+ float *dn_conv;+ float *dn_dtbias, *dn_alog, *dn_norm;+ Q38Weight ple_key, ple_value;+ float *ple_norm_key, *ple_norm_query;float *ple_norm_conv, *ple_conv;} Layer;-typedef struct { int eid; float *gate, *up, *down; uint64_t used; } Slot;+typedef struct { int eid; Q38Weight gate, up, down; uint64_t used; } Slot;typedef struct { Slot *slots; int *by_expert, n, cap; } LCache;typedef struct {Cfg c;shards S;char prefix[32];- float *embed, *lm_head;+ Q38Weight embed, lm_head;GatedResidual final_gr;Layer *L;LCache *cache;@@ -65,7 +109,10 @@ typedef struct {float *PLE_conv_state;int ple_history_len;int range_begin, range_end;+ int native_fp8, native_bf16;+ uint64_t resident_weight_bytes;double dense_load_s;+ Q38Timers timers;} Model;static float *g_last_logit;@@ -77,6 +124,18 @@ static double now_s(void) {return (double)t.tv_sec + (double)t.tv_nsec * 1e-9;}+static inline void q38_tm_add(Model *m,Q38Timer timer,double started) {+ m->timers.seconds[timer] += now_s() - started;+}++static Q38Timers q38_tm_delta(const Q38Timers *after,const Q38Timers *before) {+ Q38Timers delta={0};+ for(int i=0;i<Q38_TM_COUNT;i++)+ delta.seconds[i]=after->seconds[i]-before->seconds[i];+ delta.forwards=after->forwards-before->forwards;+ return delta;+}+#if defined(__APPLE__)static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF,&r); return r.ru_maxrss/1073741824.0; }#else@@ -106,6 +165,97 @@ static void q38_matmul(float *y, const float *x, const float *W, int S, int I, i}}+static void q38_weight_free(Q38Weight *weight) {+ if(!weight)return;+ free(weight->data);free(weight->scales);memset(weight,0,sizeof(*weight));+}++static void q38_weight_reserve(Q38Weight *weight,Q38WeightKind kind,int rows,int cols) {+ if(rows<=0||cols<=0||(int64_t)rows>INT64_MAX/cols){+ fprintf(stderr,"invalid weight geometry [%d,%d]\n",rows,cols);exit(1);+ }+ int64_t elements=(int64_t)rows*cols;+ int64_t scales=kind==Q38_WEIGHT_FP8?fp8_nblk(rows)*fp8_nblk(cols):0;+ if(weight->kind==kind&&weight->rows==rows&&weight->cols==cols&&+ weight->data&&(!scales||weight->scales))return;+ q38_weight_free(weight);+ size_t element_size=kind==Q38_WEIGHT_F32?sizeof(float):+ kind==Q38_WEIGHT_BF16?sizeof(uint16_t):+ kind==Q38_WEIGHT_FP8?sizeof(uint8_t):0;+ if(!element_size||(uint64_t)elements>SIZE_MAX/element_size||+ (scales&&(uint64_t)scales>SIZE_MAX/sizeof(float))){+ fprintf(stderr,"unsupported or oversized weight geometry [%d,%d] kind=%d\n",+ rows,cols,(int)kind);exit(1);+ }+ weight->data=malloc((size_t)elements*element_size);+ if(scales)weight->scales=(float*)malloc((size_t)scales*sizeof(float));+ if(!weight->data||(scales&&!weight->scales)){+ fprintf(stderr,"OOM allocating weight [%d,%d] kind=%d\n",rows,cols,(int)kind);exit(1);+ }+ weight->rows=rows;weight->cols=cols;weight->elements=elements;+ weight->scale_count=scales;weight->kind=kind;+}++static uint64_t q38_weight_bytes(const Q38Weight *weight) {+ uint64_t element_size=weight->kind==Q38_WEIGHT_F32?sizeof(float):+ weight->kind==Q38_WEIGHT_BF16?sizeof(uint16_t):+ weight->kind==Q38_WEIGHT_FP8?sizeof(uint8_t):0;+ return (uint64_t)weight->elements*element_size++ (uint64_t)weight->scale_count*sizeof(float);+}++/* Native BF16 storage with FP32 activations and accumulation. This deliberately+ * does not round activations to BF16 or use BF16 dot-product instructions: it is+ * the storage-equivalent form of the existing st_read_f32 reference. */+static void q38_matmul_bf16(float *y,const float *x,const uint16_t *W,+ int S,int I,int O) {+ #pragma omp parallel for schedule(static)+ for(int o=0;o<O;o++){+ const uint16_t *w=W+(int64_t)o*I;+ for(int s=0;s<S;s++){+ const float *xs=x+(int64_t)s*I;float a=0.f;+ for(int i=0;i<I;i++)a+=xs[i]*bf16_to_f32(w[i]);+ y[(int64_t)s*O+o]=a;+ }+ }+}++static void q38_weight_matmul(float *y,const float *x,const Q38Weight *weight,+ int S,int I,int O) {+ if(!weight||weight->rows!=O||weight->cols!=I||!weight->data){+ fprintf(stderr,"invalid matmul weight: have [%d,%d] kind=%d, need [%d,%d]\n",+ weight?weight->rows:0,weight?weight->cols:0,+ weight?(int)weight->kind:0,O,I);exit(1);+ }+ if(weight->kind==Q38_WEIGHT_F32)+ q38_matmul(y,x,(const float*)weight->data,S,I,O);+ else if(weight->kind==Q38_WEIGHT_BF16)+ q38_matmul_bf16(y,x,(const uint16_t*)weight->data,S,I,O);+ else if(weight->kind==Q38_WEIGHT_FP8&&weight->scales)+ matmul_fp8(y,x,(const uint8_t*)weight->data,weight->scales,S,I,O);+ else {fprintf(stderr,"unsupported matmul weight kind %d\n",(int)weight->kind);exit(1);}+}++static void q38_weight_row(const Q38Weight *weight,int row,float *out) {+ if(!weight||row<0||row>=weight->rows||!weight->data){+ fprintf(stderr,"invalid weight row %d\n",row);exit(1);+ }+ if(weight->kind==Q38_WEIGHT_F32)+ memcpy(out,(const float*)weight->data+(int64_t)row*weight->cols,+ (size_t)weight->cols*sizeof(float));+ else if(weight->kind==Q38_WEIGHT_BF16){+ const uint16_t *src=(const uint16_t*)weight->data+(int64_t)row*weight->cols;+ for(int i=0;i<weight->cols;i++)out[i]=bf16_to_f32(src[i]);+ } else {fprintf(stderr,"weight kind %d has no dense row view\n",(int)weight->kind);exit(1);}+}++static void q38_dense_matmul(Model *m,float *y,const float *x,const Q38Weight *weight,+ int S,int I,int O) {+ double started=now_s();+ q38_weight_matmul(y,x,weight,S,I,O);+ q38_tm_add(m,Q38_TM_DENSE_MATMUL,started);+}+static inline float q38_sigmoid(float x) {if (x >= 0.f) { float z=expf(-x); return 1.f/(1.f+z); }float z=expf(x); return z/(1.f+z);@@ -353,6 +503,37 @@ static float *q38_load_tensor(Model *m,const char *name,int64_t want) {float *p=falloc(want); st_read_f32(&m->S,name,p,1); return p;}+static int q38_env_bool(const char *name,int default_value) {+ const char *value=getenv(name);if(!value||!*value)return default_value;+ if(value[0]=='0'&&!value[1])return 0;+ if(value[0]=='1'&&!value[1])return 1;+ fprintf(stderr,"%s must be exactly 0 or 1\n",name);exit(1);+}++static Q38Weight q38_load_weight(Model *m,const char *name,int rows,int cols) {+ st_tensor *tensor=st_find(&m->S,name);Q38Weight weight={0};+ if(!tensor){fprintf(stderr,"missing %s\n",name);exit(1);}+ if(tensor->rank!=2||tensor->shape[0]!=rows||tensor->shape[1]!=cols||+ tensor->numel!=(int64_t)rows*cols){+ fprintf(stderr,"%s: invalid matrix shape, expected [%d,%d]\n",name,rows,cols);exit(1);+ }+ if(tensor->dtype==0&&m->native_bf16){+ q38_weight_reserve(&weight,Q38_WEIGHT_BF16,rows,cols);+ if(tensor->nbytes!=weight.elements*(int64_t)sizeof(uint16_t)){+ fprintf(stderr,"%s: invalid BF16 byte count\n",name);exit(1);+ }+ st_read_raw_cap(&m->S,name,weight.data,tensor->nbytes,1);+ } else {+ if(tensor->dtype<0||tensor->dtype>2){+ fprintf(stderr,"%s: unsupported resident dtype %s\n",name,st_dtype_name(tensor->dtype));exit(1);+ }+ q38_weight_reserve(&weight,Q38_WEIGHT_F32,rows,cols);+ st_read_f32(&m->S,name,(float*)weight.data,1);+ }+ m->resident_weight_bytes+=q38_weight_bytes(&weight);+ return weight;+}+static void q38_name(Model *m,char *out,size_t cap,int layer,const char *suffix) {snprintf(out,cap,"%s.layers.%d.%s",m->prefix,layer,suffix);}@@ -361,20 +542,18 @@ static void q38_load_gr(Model *m,GatedResidual *g,int layer,const char *kind,intCfg *c=&m->c; char nm[320],base[180];if(layer>=0) snprintf(base,sizeof base,"layers.%d.%s",layer,kind); else snprintf(base,sizeof base,"hyper_connection_mixer");snprintf(nm,sizeof nm,"%s.%s.hc_norm.weight",m->prefix,base); g->norm=q38_load_tensor(m,nm,c->hc_width);- snprintf(nm,sizeof nm,"%s.%s.input_mix_weight_down.weight",m->prefix,base); g->down=q38_load_tensor(m,nm,(int64_t)c->hc_rank*c->hc_width);- snprintf(nm,sizeof nm,"%s.%s.input_mix_weight_up.weight",m->prefix,base); g->up=q38_load_tensor(m,nm,(int64_t)c->hc_width*c->hc_rank);- if(inject){snprintf(nm,sizeof nm,"%s.%s.block_inject_weight.weight",m->prefix,base);g->inject=q38_load_tensor(m,nm,(int64_t)c->hc_count*c->hc_width);}+ snprintf(nm,sizeof nm,"%s.%s.input_mix_weight_down.weight",m->prefix,base); g->down=q38_load_weight(m,nm,c->hc_rank,c->hc_width);+ snprintf(nm,sizeof nm,"%s.%s.input_mix_weight_up.weight",m->prefix,base); g->up=q38_load_weight(m,nm,c->hc_width,c->hc_rank);+ if(inject){snprintf(nm,sizeof nm,"%s.%s.block_inject_weight.weight",m->prefix,base);g->inject=q38_load_weight(m,nm,c->hc_count,c->hc_width);}}static void q38_load_ple(Model *m,Layer *l) {Cfg *c=&m->c; int i=c->ple_layer; char nm[320];+ q38_name(m,nm,sizeof nm,i,"ple.key_proj.weight");l->ple_key=q38_load_weight(m,nm,c->hc_width,c->ple_dim);+ q38_name(m,nm,sizeof nm,i,"ple.value_proj.weight");l->ple_value=q38_load_weight(m,nm,c->hidden,c->ple_dim);#define PL(field,suf,n) q38_name(m,nm,sizeof nm,i,"ple." suf); l->field=q38_load_tensor(m,nm,(n))- PL(ple_key,"key_proj.weight",(int64_t)c->hc_width*c->ple_dim);- PL(ple_value,"value_proj.weight",(int64_t)c->hidden*c->ple_dim);- PL(ple_norm_key,"norm_key.weight",c->hc_width);- PL(ple_norm_query,"norm_query.weight",c->hc_width);- PL(ple_norm_conv,"norm_conv.weight",c->hc_width);- PL(ple_conv,"conv1d.weight",(int64_t)c->hc_width*c->ple_convk);+ PL(ple_norm_key,"norm_key.weight",c->hc_width);PL(ple_norm_query,"norm_query.weight",c->hc_width);+ PL(ple_norm_conv,"norm_conv.weight",c->hc_width);PL(ple_conv,"conv1d.weight",(int64_t)c->hc_width*c->ple_convk);#undef PLconst char *bufs[]={"layer_multipliers","ngram_heads_vocab_sizes","ngram_heads_offsets"};int64_t *dsts[]={m->ple_multipliers,m->ple_head_vocab,m->ple_head_offset};@@ -442,6 +621,8 @@ static void model_init_range(Model *m,const char *snap,int cap,int bits,int layer_begin,int layer_end,int load_boundaries,int allocate_state) {(void)bits; memset(m,0,sizeof(*m)); double t0=now_s();+ m->native_fp8=q38_env_bool("Q38_NATIVE_FP8",1);+ m->native_bf16=q38_env_bool("Q38_NATIVE_BF16",1);q38_load_cfg(&m->c,snap); q38_validate_cfg(&m->c); st_init(&m->S,snap);Cfg *c=&m->c; char nm[320];if(st_has(&m->S,"model.language_model.embed_tokens.weight")) snprintf(m->prefix,sizeof m->prefix,"model.language_model");@@ -451,8 +632,8 @@ static void model_init_range(Model *m,const char *snap,int cap,int bits,if(layer_begin<0||layer_begin>=layer_end||layer_end>c->layers){fprintf(stderr,"invalid Qwen3.8 layer range [%d,%d)\n",layer_begin,layer_end);exit(1);}m->range_begin=layer_begin; m->range_end=layer_end;if(load_boundaries){- snprintf(nm,sizeof nm,"%s.embed_tokens.weight",m->prefix); m->embed=q38_load_tensor(m,nm,(int64_t)c->vocab*c->hidden);- m->lm_head=q38_load_tensor(m,"lm_head.weight",(int64_t)c->vocab*c->hidden);+ snprintf(nm,sizeof nm,"%s.embed_tokens.weight",m->prefix); m->embed=q38_load_weight(m,nm,c->vocab,c->hidden);+ m->lm_head=q38_load_weight(m,"lm_head.weight",c->vocab,c->hidden);q38_load_gr(m,&m->final_gr,-1,NULL,0);}m->L=(Layer*)calloc((size_t)c->layers,sizeof(Layer));@@ -460,64 +641,71 @@ static void model_init_range(Model *m,const char *snap,int cap,int bits,if(!m->L||!m->cache){fprintf(stderr,"OOM model metadata\n");exit(1);}for(int i=layer_begin;i<layer_end;i++){Layer *l=&m->L[i]; q38_load_gr(m,&l->attn_gr,i,"attn_hyper_connection",1); q38_load_gr(m,&l->mlp_gr,i,"mlp_hyper_connection",1);- #define LD(field,suf,n) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_tensor(m,nm,(n))- LD(router,"mlp.gate.weight",(int64_t)c->experts*c->hidden);- LD(sh_g,"mlp.shared_expert.gate_proj.weight",(int64_t)c->shared_inter*c->hidden);- LD(sh_u,"mlp.shared_expert.up_proj.weight",(int64_t)c->shared_inter*c->hidden);- LD(sh_d,"mlp.shared_expert.down_proj.weight",(int64_t)c->hidden*c->shared_inter);- LD(sh_gate,"mlp.shared_expert_gate.weight",c->hidden);+ #define WLD(field,suf,o,in) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_weight(m,nm,(o),(in))+ #define VLD(field,suf,n) q38_name(m,nm,sizeof nm,i,suf); l->field=q38_load_tensor(m,nm,(n))+ WLD(router,"mlp.gate.weight",c->experts,c->hidden);+ WLD(sh_g,"mlp.shared_expert.gate_proj.weight",c->shared_inter,c->hidden);+ WLD(sh_u,"mlp.shared_expert.up_proj.weight",c->shared_inter,c->hidden);+ WLD(sh_d,"mlp.shared_expert.down_proj.weight",c->hidden,c->shared_inter);+ VLD(sh_gate,"mlp.shared_expert_gate.weight",c->hidden);if(c->is_attn[i]){- LD(q,"self_attn.q_proj.weight",(int64_t)c->q_heads*c->head_dim*2*c->hidden);- LD(k,"self_attn.k_proj.weight",(int64_t)c->kv_heads*c->head_dim*c->hidden);- LD(v,"self_attn.v_proj.weight",(int64_t)c->kv_heads*c->head_dim*c->hidden);- LD(o,"self_attn.o_proj.weight",(int64_t)c->hidden*c->q_heads*c->head_dim);- LD(qn,"self_attn.q_norm.weight",c->head_dim);- LD(kn,"self_attn.k_norm.weight",c->head_dim);- LD(idx_qk,"self_attn.indexer.index_qk_proj.weight",(int64_t)(c->idx_qheads+c->idx_kheads)*c->idx_dim*c->hidden);- LD(idx_qn,"self_attn.indexer.q_layernorm.weight",c->idx_dim);- LD(idx_kn,"self_attn.indexer.k_layernorm.weight",c->idx_dim);+ WLD(q,"self_attn.q_proj.weight",c->q_heads*c->head_dim*2,c->hidden);+ WLD(k,"self_attn.k_proj.weight",c->kv_heads*c->head_dim,c->hidden);+ WLD(v,"self_attn.v_proj.weight",c->kv_heads*c->head_dim,c->hidden);+ WLD(o,"self_attn.o_proj.weight",c->hidden,c->q_heads*c->head_dim);+ VLD(qn,"self_attn.q_norm.weight",c->head_dim);VLD(kn,"self_attn.k_norm.weight",c->head_dim);+ WLD(idx_qk,"self_attn.indexer.index_qk_proj.weight",(c->idx_qheads+c->idx_kheads)*c->idx_dim,c->hidden);+ VLD(idx_qn,"self_attn.indexer.q_layernorm.weight",c->idx_dim);VLD(idx_kn,"self_attn.indexer.k_layernorm.weight",c->idx_dim);} else {int vd=c->dn_vheads*c->dn_vdim;- LD(dn_qkv,"linear_attn.in_proj_qkv.weight",(int64_t)c->dn_conv_dim*c->hidden);- LD(dn_z,"linear_attn.in_proj_z.weight",(int64_t)vd*c->hidden);- LD(dn_b,"linear_attn.in_proj_b.weight",(int64_t)c->dn_vheads*c->hidden);- LD(dn_a,"linear_attn.in_proj_a.weight",(int64_t)c->dn_vheads*c->hidden);- LD(dn_conv,"linear_attn.conv1d.weight",(int64_t)c->dn_conv_dim*c->dn_convk);- LD(dn_dtbias,"linear_attn.dt_bias",c->dn_vheads);- LD(dn_alog,"linear_attn.A_log",c->dn_vheads);- LD(dn_norm,"linear_attn.norm.weight",c->dn_vdim);- LD(dn_out,"linear_attn.out_proj.weight",(int64_t)c->hidden*vd);+ WLD(dn_qkv,"linear_attn.in_proj_qkv.weight",c->dn_conv_dim,c->hidden);+ WLD(dn_z,"linear_attn.in_proj_z.weight",vd,c->hidden);+ WLD(dn_b,"linear_attn.in_proj_b.weight",c->dn_vheads,c->hidden);+ WLD(dn_a,"linear_attn.in_proj_a.weight",c->dn_vheads,c->hidden);+ VLD(dn_conv,"linear_attn.conv1d.weight",(int64_t)c->dn_conv_dim*c->dn_convk);+ VLD(dn_dtbias,"linear_attn.dt_bias",c->dn_vheads);VLD(dn_alog,"linear_attn.A_log",c->dn_vheads);+ VLD(dn_norm,"linear_attn.norm.weight",c->dn_vdim);+ WLD(dn_out,"linear_attn.out_proj.weight",c->hidden,vd);}- #undef LD+ #undef WLD+ #undef VLDLCache *lc=&m->cache[i]; lc->cap=cap; lc->slots=(Slot*)calloc((size_t)cap,sizeof(Slot)); lc->by_expert=(int*)malloc((size_t)c->experts*sizeof(int));if(!lc->slots||!lc->by_expert){fprintf(stderr,"OOM expert cache\n");exit(1);} for(int e=0;e<c->experts;e++)lc->by_expert[e]=-1;}if(c->ple_layer>=layer_begin&&c->ple_layer<layer_end) q38_load_ple(m,&m->L[c->ple_layer]);if(allocate_state) q38_alloc_state(m);m->dense_load_s=now_s()-t0;- fprintf(stderr,"[qwen38] native text weights: prefix=%s, %d layers, PLE=%d, cache=%d/layer\n",m->prefix,c->layers,c->ple_layer,cap);+ fprintf(stderr,"[qwen38] native text weights: prefix=%s, %d layers, PLE=%d, cache=%d/layer, "+ "FP8=%s, BF16=%s (resident matrices %.2f GiB)\n",m->prefix,c->layers,c->ple_layer,cap,+ m->native_fp8?"native":"expanded-f32",m->native_bf16?"native":"expanded-f32",+ m->resident_weight_bytes/1073741824.0);}static void model_init(Model *m,const char *snap,int cap,int bits) {model_init_range(m,snap,cap,bits,0,0,1,1);}-static void q38_gr_read(const Cfg *c,const GatedResidual *g,const float *hyper,+static void q38_gr_read(Model *m,const GatedResidual *g,const float *hyper,int S,float *mixed,float *inject) {+ const Cfg *c=&m->c;int H=c->hidden,W=c->hc_width,C=c->hc_count,R=c->hc_rank;+ if(C<=0||H<=0||W!=C*H){fprintf(stderr,"invalid gated-residual width\n");exit(1);}float *norm=falloc((int64_t)S*W),*low=falloc((int64_t)S*R),*mix=falloc((int64_t)S*W);+ /* Makes the initialized-input invariant visible to aggressive interprocedural+ * warning analysis; every element is overwritten by q38_rms0 below. */+ memset(norm,0,(size_t)S*W*sizeof(float));for(int s=0;s<S;s++) for(int b=0;b<C;b++)q38_rms0(norm+(int64_t)s*W+(int64_t)b*H,hyper+(int64_t)s*W+(int64_t)b*H,g->norm+(int64_t)b*H,H,c->eps);- q38_matmul(low,norm,g->down,S,W,R);+ q38_dense_matmul(m,low,norm,&g->down,S,W,R);for(int64_t z=0;z<(int64_t)S*R;z++) low[z]=q38_silu(low[z]/C);- q38_matmul(mix,low,g->up,S,R,W);+ q38_dense_matmul(m,mix,low,&g->up,S,R,W);for(int s=0;s<S;s++) for(int d=0;d<H;d++){float v=0.f;for(int b=0;b<C;b++) v+=q38_sigmoid(mix[(int64_t)s*W+(int64_t)b*H+d])*norm[(int64_t)s*W+(int64_t)b*H+d];mixed[(int64_t)s*H+d]=v/C;}if(inject){- q38_matmul(inject,norm,g->inject,S,W,C);+ q38_dense_matmul(m,inject,norm,&g->inject,S,W,C);for(int64_t z=0;z<(int64_t)S*C;z++) inject[z]=2.f*q38_sigmoid(inject[z]/C);}free(norm);free(low);free(mix);@@ -531,46 +719,91 @@ static void q38_gr_apply(const Cfg *c,float *hyper,const float *block,const floa}}-static void q38_decode_fp8(Model *m,const char *wn,const char *sn,float *out,int O,int I) {+static void q38_load_fp8_expert_weight(Model *m,const char *wn,const char *sn,+ Q38Weight *out,int O,int I) {st_tensor *w=st_find(&m->S,wn),*sc=st_find(&m->S,sn);int nb_o=(O+127)/128,nb_i=(I+127)/128;if(!w||w->dtype!=4||w->rank!=2||w->shape[0]!=O||w->shape[1]!=I||- !sc||sc->rank!=2||sc->shape[0]!=nb_o||sc->shape[1]!=nb_i){+ w->nbytes!=(int64_t)O*I||!sc||sc->dtype<0||sc->dtype>2||+ sc->rank!=2||sc->shape[0]!=nb_o||sc->shape[1]!=nb_i||+ sc->numel!=(int64_t)nb_o*nb_i){fprintf(stderr,"invalid block-FP8 expert matrix %s / %s\n",wn,sn);exit(1);}- uint8_t *raw=(uint8_t*)malloc((size_t)O*I); float *scale=falloc((int64_t)nb_o*nb_i);- if(!raw){fprintf(stderr,"OOM FP8 expert read\n");exit(1);}- st_read_raw_cap(&m->S,wn,raw,(int64_t)O*I,1); st_read_f32(&m->S,sn,scale,1);+ if(m->native_fp8){+ q38_weight_reserve(out,Q38_WEIGHT_FP8,O,I);+ double started=now_s();+ st_read_raw_cap(&m->S,wn,out->data,w->nbytes,1);+ st_read_f32(&m->S,sn,out->scales,1);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);+ return;+ }+ q38_weight_reserve(out,Q38_WEIGHT_F32,O,I);+ uint8_t *raw=(uint8_t*)malloc((size_t)w->nbytes);float *scale=falloc(sc->numel);+ if(!raw){fprintf(stderr,"OOM FP8 expert staging\n");exit(1);}+ double started=now_s();+ st_read_raw_cap(&m->S,wn,raw,w->nbytes,1);st_read_f32(&m->S,sn,scale,1);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started); started=now_s();+ float *decoded=(float*)out->data;#pragma omp parallel for schedule(static)- for(int o=0;o<O;o++) for(int i=0;i<I;i++) out[(int64_t)o*I+i]=e4m3_decode(raw[(int64_t)o*I+i])*scale[(o/128)*nb_i+i/128];+ for(int o=0;o<O;o++)for(int i=0;i<I;i++)+ decoded[(int64_t)o*I+i]=e4m3_decode(raw[(int64_t)o*I+i])*+ scale[(o/128)*nb_i+i/128];+ q38_tm_add(m,Q38_TM_FP8_EXPAND,started);free(raw);free(scale);}+static void q38_load_expert_weight(Model *m,const char *name,Q38Weight *out,+ int O,int I) {+ st_tensor *tensor=st_find(&m->S,name);+ if(!tensor||tensor->rank!=2||tensor->shape[0]!=O||tensor->shape[1]!=I||+ tensor->numel!=(int64_t)O*I||tensor->dtype<0||tensor->dtype>2){+ fprintf(stderr,"invalid expert matrix %s\n",name);exit(1);+ }+ Q38WeightKind kind=m->native_bf16&&tensor->dtype==0?Q38_WEIGHT_BF16:Q38_WEIGHT_F32;+ q38_weight_reserve(out,kind,O,I);double started=now_s();+ if(kind==Q38_WEIGHT_BF16)+ st_read_raw_cap(&m->S,name,out->data,tensor->nbytes,1);+ else st_read_f32(&m->S,name,(float*)out->data,1);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);+}++static void q38_load_expert_slice(Model *m,const char *name,const st_tensor *tensor,+ int64_t element_offset,Q38Weight *out,int O,int I) {+ if(!tensor||tensor->dtype<0||tensor->dtype>2||element_offset<0||+ element_offset>tensor->numel-(int64_t)O*I){+ fprintf(stderr,"invalid expert slice %s\n",name);exit(1);+ }+ Q38WeightKind kind=m->native_bf16&&tensor->dtype==0?Q38_WEIGHT_BF16:Q38_WEIGHT_F32;+ q38_weight_reserve(out,kind,O,I);double started=now_s();+ if(kind==Q38_WEIGHT_BF16){+ int64_t bytes=(int64_t)O*I*(int64_t)sizeof(uint16_t);+ st_read_slice_raw_cap(&m->S,name,element_offset*(int64_t)sizeof(uint16_t),+ bytes,out->data,bytes,1);+ } else st_read_slice_f32(&m->S,name,element_offset,(int64_t)O*I,(float*)out->data,1);+ q38_tm_add(m,Q38_TM_EXPERT_READ,started);+}+static void q38_load_expert(Model *m,int layer,int eid,Slot *s) {Cfg *c=&m->c; int H=c->hidden,I=c->inter; char nm[320],sn[340];- if(!s->gate){- float *all=falloc((int64_t)3*I*H); s->gate=all;s->up=all+(int64_t)I*H;s->down=all+(int64_t)2*I*H;- }q38_name(m,nm,sizeof nm,layer,"mlp.experts.gate_up_proj");if(st_has(&m->S,nm)){st_tensor *t=st_find(&m->S,nm);if(t->rank!=3||t->shape[0]!=c->experts||t->shape[1]!=2*I||t->shape[2]!=H){fprintf(stderr,"invalid %s\n",nm);exit(1);}- st_read_slice_f32(&m->S,nm,(int64_t)eid*2*I*H,(int64_t)2*I*H,s->gate,1);+ int64_t base=(int64_t)eid*2*I*H;+ q38_load_expert_slice(m,nm,t,base,&s->gate,I,H);+ q38_load_expert_slice(m,nm,t,base+(int64_t)I*H,&s->up,I,H);q38_name(m,nm,sizeof nm,layer,"mlp.experts.down_proj"); t=st_find(&m->S,nm);if(!t||t->rank!=3||t->shape[0]!=c->experts||t->shape[1]!=H||t->shape[2]!=I){fprintf(stderr,"invalid %s\n",nm);exit(1);}- st_read_slice_f32(&m->S,nm,(int64_t)eid*H*I,(int64_t)H*I,s->down,1);+ q38_load_expert_slice(m,nm,t,(int64_t)eid*H*I,&s->down,H,I);return;}- const char *kind[3]={"gate_proj","up_proj","down_proj"}; float *dst[3]={s->gate,s->up,s->down};+ const char *kind[3]={"gate_proj","up_proj","down_proj"};Q38Weight *dst[3]={&s->gate,&s->up,&s->down};int os[3]={I,I,H},is[3]={H,H,I};for(int k=0;k<3;k++){char suf[192]; snprintf(suf,sizeof suf,"mlp.experts.%d.%s.weight",eid,kind[k]);q38_name(m,nm,sizeof nm,layer,suf);st_tensor *t=st_find(&m->S,nm); if(!t){fprintf(stderr,"missing %s\n",nm);exit(1);}- if(t->dtype==4){snprintf(sn,sizeof sn,"%s_scale_inv",nm);q38_decode_fp8(m,nm,sn,dst[k],os[k],is[k]);}- else {- if(t->rank!=2||t->shape[0]!=os[k]||t->shape[1]!=is[k]){fprintf(stderr,"invalid %s\n",nm);exit(1);}- st_read_f32(&m->S,nm,dst[k],1);- }+ if(t->dtype==4){snprintf(sn,sizeof sn,"%s_scale_inv",nm);q38_load_fp8_expert_weight(m,nm,sn,dst[k],os[k],is[k]);}+ else q38_load_expert_weight(m,nm,dst[k],os[k],is[k]);}}@@ -609,6 +842,7 @@ static int64_t q38_hash_row(Model *m,int head,int ngram,int64_t cur,int64_t p1,i}static void q38_ple(Model *m,const int *ids,int S,const float *hyper,float *out) {+ double phase_started=now_s();Cfg *c=&m->c; Layer *l=&m->L[c->ple_layer]; int H=c->hidden,C=c->hc_count,W=c->hc_width,E=c->ple_dim;float *emb=falloc(E),*keys=falloc(W),*value=falloc(H),*kn=falloc(W),*qn=falloc(W),*gated=falloc(W),*norm=falloc(W);int state_len=(c->ple_convk-1)*c->ngram_size; float *ring=m->PLE_conv_state;@@ -619,7 +853,7 @@ static void q38_ple(Model *m,const int *ids,int S,const float *hyper,float *out)int ng=h<c->heads_per_ngram?2:3; int64_t row=q38_hash_row(m,h,ng,ids[s],p1,p2);q38_ple_row(m,row,emb+(int64_t)h*c->ngram_head_dim);}- q38_matmul(keys,emb,l->ple_key,1,E,W);q38_matmul(value,emb,l->ple_value,1,E,H);+ q38_dense_matmul(m,keys,emb,&l->ple_key,1,E,W);q38_dense_matmul(m,value,emb,&l->ple_value,1,E,H);for(int b=0;b<C;b++){q38_rms0(kn+(int64_t)b*H,keys+(int64_t)b*H,l->ple_norm_key+(int64_t)b*H,H,c->eps);q38_rms0(qn+(int64_t)b*H,hyper+(int64_t)s*W+(int64_t)b*H,l->ple_norm_query+(int64_t)b*H,H,c->eps);@@ -640,9 +874,11 @@ static void q38_ple(Model *m,const int *ids,int S,const float *hyper,float *out)else {m->ple_history[0]=m->ple_history[1];m->ple_history[1]=ids[s];}}free(emb);free(keys);free(value);free(kn);free(qn);free(gated);free(norm);+ q38_tm_add(m,Q38_TM_PLE,phase_started);}static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,float *out) {+ double phase_started=now_s();Cfg *c=&m->c; int H=c->hidden,VH=c->dn_vheads,KH=c->dn_kheads,KD=c->dn_kdim,VD=c->dn_vdim;int CD=c->dn_conv_dim,CK=c->dn_convk,V=VH*VD,K=KH*KD,rep=VH/KH;float *qkv=falloc(CD),*z=falloc(V),*bb=falloc(VH),*aa=falloc(VH),*conv=falloc(CD);@@ -650,8 +886,8 @@ static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,floatfloat *rec=m->DN_rec[layer],*ring=m->DN_conv[layer];for(int s=0;s<S;s++){const float *xs=x+(int64_t)s*H;- q38_matmul(qkv,xs,l->dn_qkv,1,H,CD);q38_matmul(z,xs,l->dn_z,1,H,V);- q38_matmul(bb,xs,l->dn_b,1,H,VH);q38_matmul(aa,xs,l->dn_a,1,H,VH);+ q38_dense_matmul(m,qkv,xs,&l->dn_qkv,1,H,CD);q38_dense_matmul(m,z,xs,&l->dn_z,1,H,V);+ q38_dense_matmul(m,bb,xs,&l->dn_b,1,H,VH);q38_dense_matmul(m,aa,xs,&l->dn_a,1,H,VH);for(int d=0;d<CD;d++){float a=l->dn_conv[(int64_t)d*CK+CK-1]*qkv[d];float *r=ring+(int64_t)d*(CK-1);for(int j=0;j<CK-1;j++)a+=l->dn_conv[(int64_t)d*CK+j]*r[j];conv[d]=q38_silu(a);@@ -677,9 +913,10 @@ static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,floatfor(int v0=0;v0<VD;v0++){float a=0.f;for(int d=0;d<KD;d++)a+=qh[d]*state[(int64_t)d*VD+v0];core[(int64_t)h*VD+v0]=a;}}for(int h=0;h<VH;h++)q38_rmsg(norm+(int64_t)h*VD,core+(int64_t)h*VD,z+(int64_t)h*VD,l->dn_norm,VD,c->eps,1);- q38_matmul(out+(int64_t)s*H,norm,l->dn_out,1,V,H);+ q38_dense_matmul(m,out+(int64_t)s*H,norm,&l->dn_out,1,V,H);}free(qkv);free(z);free(bb);free(aa);free(conv);free(q);free(k);free(core);free(norm);+ q38_tm_add(m,Q38_TM_DELTANET,phase_started);}typedef struct { float score; int block; } Q38Block;@@ -693,8 +930,8 @@ static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int pint IQ=c->idx_qheads,ID=c->idx_dim,R=c->idx_ratio,maxsel=c->idx_budget+R-1;float *qp=falloc((int64_t)S*QH*2*D),*kp=falloc((int64_t)S*KVH*D),*vp=falloc((int64_t)S*KVH*D);float *ip=falloc((int64_t)S*(IQ+c->idx_kheads)*ID);- q38_matmul(qp,x,l->q,S,H,QH*2*D);q38_matmul(kp,x,l->k,S,H,KVH*D);q38_matmul(vp,x,l->v,S,H,KVH*D);- q38_matmul(ip,x,l->idx_qk,S,H,(IQ+c->idx_kheads)*ID);+ q38_dense_matmul(m,qp,x,&l->q,S,H,QH*2*D);q38_dense_matmul(m,kp,x,&l->k,S,H,KVH*D);q38_dense_matmul(m,vp,x,&l->v,S,H,KVH*D);+ q38_dense_matmul(m,ip,x,&l->idx_qk,S,H,(IQ+c->idx_kheads)*ID);for(int s=0;s<S;s++){int pos=pos_base+s;for(int h=0;h<KVH;h++){@@ -709,6 +946,7 @@ static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int pif(!selected){fprintf(stderr,"OOM QSA selection\n");exit(1);}for(int s=0;s<S;s++){int pos=pos_base+s,visible=pos+1,blocks=visible/R,tail=blocks*R;+ double phase_started=now_s();for(int h=0;h<IQ;h++){float *qh=qidx+(int64_t)h*ID;memcpy(qh,ip+(int64_t)s*(IQ+1)*ID+(int64_t)h*ID,(size_t)ID*sizeof(float));q38_rms0(qh,qh,l->idx_qn,ID,c->eps);q38_rope(qh,ID,c->rotary_dim,pos,c->theta);}int take=blocks<c->idx_budget/R?blocks:c->idx_budget/R,nsel=0;Q38Block *rank=blocks?(Q38Block*)malloc((size_t)blocks*sizeof(Q38Block)):NULL;@@ -721,6 +959,7 @@ static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int pif(blocks)qsort(rank,(size_t)blocks,sizeof(Q38Block),q38_block_desc);for(int z=0;z<take;z++)for(int r=0;r<R;r++)selected[nsel++]=rank[z].block*R+r;for(int t=tail;t<visible;t++)selected[nsel++]=t;free(rank);+ q38_tm_add(m,Q38_TM_QSA_INDEX,phase_started); phase_started=now_s();for(int h=0;h<QH;h++){float *qraw=qp+(int64_t)s*QH*2*D+(int64_t)h*2*D;float *qh=falloc(D);memcpy(qh,qraw,(size_t)D*sizeof(float));q38_rms0(qh,qh,l->qn,D,c->eps);q38_rope(qh,D,c->rotary_dim,pos,c->theta);@@ -731,8 +970,9 @@ static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int pfor(int j=0;j<nsel;j++){float a=score[j]/den;const float *vh=m->V[layer]+((int64_t)khidx*m->kv_cap+selected[j])*D;for(int d=0;d<D;d++)oh[d]+=a*vh[d];}for(int d=0;d<D;d++)oh[d]*=q38_sigmoid(qraw[D+d]);free(qh);free(score);}+ q38_tm_add(m,Q38_TM_QSA_ATTENTION,phase_started);}- q38_matmul(out,heads,l->o,S,QH*D,H);+ q38_dense_matmul(m,out,heads,&l->o,S,QH*D,H);free(qp);free(kp);free(vp);free(ip);free(heads);free(qidx);free(pool);free(selected);}@@ -742,7 +982,7 @@ static void q38_moe(Model *m,Layer *l,int layer,const float *x,int S,float *out)float *eg=falloc(I),*eu=falloc(I),*eh=falloc(I),*eo=falloc(H);for(int s=0;s<S;s++){const float *xs=x+(int64_t)s*H;float *ys=out+(int64_t)s*H;memset(ys,0,(size_t)H*sizeof(float));- q38_matmul(logits,xs,l->router,1,H,E);float mx=logits[0];for(int e=1;e<E;e++)if(logits[e]>mx)mx=logits[e];+ q38_dense_matmul(m,logits,xs,&l->router,1,H,E);float mx=logits[0];for(int e=1;e<E;e++)if(logits[e]>mx)mx=logits[e];double all=0;for(int e=0;e<E;e++){logits[e]=expf(logits[e]-mx);all+=logits[e];}int idx[Q38_MAX_TOPK];float val[Q38_MAX_TOPK];for(int z=0;z<K;z++){int best=-1;float bv=-1.f;for(int e=0;e<E;e++){int used=0;for(int j=0;j<z;j++)if(idx[j]==e)used=1;if(!used&&logits[e]>bv){bv=logits[e];best=e;}}idx[z]=rt_router_pick(best,z,E,layer);val[z]=logits[idx[z]];}@@ -751,14 +991,18 @@ static void q38_moe(Model *m,Layer *l,int layer,const float *x,int S,float *out)for(int z=0;z<K;z++) route_gates[z]=(float)(val[z]/den);rt_route(layer,s,idx,route_gates,K); /* shared counts + post-normalization trace */for(int z=0;z<K;z++){- Slot *ex=q38_expert_get(m,layer,idx[z]);q38_matmul(eg,xs,ex->gate,1,H,I);q38_matmul(eu,xs,ex->up,1,H,I);- for(int j=0;j<I;j++)eh[j]=q38_silu(eg[j])*eu[j];q38_matmul(eo,eh,ex->down,1,I,H);+ Slot *ex=q38_expert_get(m,layer,idx[z]); double phase_started=now_s();+ q38_weight_matmul(eg,xs,&ex->gate,1,H,I);q38_weight_matmul(eu,xs,&ex->up,1,H,I);+ for(int j=0;j<I;j++)eh[j]=q38_silu(eg[j])*eu[j];q38_weight_matmul(eo,eh,&ex->down,1,I,H);for(int d=0;d<H;d++)ys[d]+=route_gates[z]*eo[d];+ q38_tm_add(m,Q38_TM_ROUTED_EXPERT,phase_started);}- q38_matmul(sg,xs,l->sh_g,1,H,SI);q38_matmul(su,xs,l->sh_u,1,H,SI);- for(int j=0;j<SI;j++)sh[j]=q38_silu(sg[j])*su[j];q38_matmul(shared,sh,l->sh_d,1,SI,H);+ double phase_started=now_s();+ q38_weight_matmul(sg,xs,&l->sh_g,1,H,SI);q38_weight_matmul(su,xs,&l->sh_u,1,H,SI);+ for(int j=0;j<SI;j++)sh[j]=q38_silu(sg[j])*su[j];q38_weight_matmul(shared,sh,&l->sh_d,1,SI,H);float gate=0.f;for(int d=0;d<H;d++)gate+=xs[d]*l->sh_gate[d];gate=q38_sigmoid(gate);for(int d=0;d<H;d++)ys[d]+=gate*shared[d];+ q38_tm_add(m,Q38_TM_SHARED_EXPERT,phase_started);}rt_trace_end();free(logits);free(sg);free(su);free(sh);free(shared);free(eg);free(eu);free(eh);free(eo);@@ -798,11 +1042,11 @@ static void q38_layers_forward_range(Model *m,float *hyper,const int *ids,for(int64_t z=0;z<(int64_t)S*W;z++) hyper[z]+=ple[z];free(ple);}- q38_gr_read(c,&l->attn_gr,hyper,S,mixed,inject);+ q38_gr_read(m,&l->attn_gr,hyper,S,mixed,inject);if(c->is_attn[i]) q38_attention(m,l,i,mixed,S,pos_base,block);else q38_deltanet(m,l,i,mixed,S,block);q38_gr_apply(c,hyper,block,inject,S);- q38_gr_read(c,&l->mlp_gr,hyper,S,mixed,inject);+ q38_gr_read(m,&l->mlp_gr,hyper,S,mixed,inject);q38_moe(m,l,i,mixed,S,block);q38_gr_apply(c,hyper,block,inject,S);}@@ -811,37 +1055,64 @@ static void q38_layers_forward_range(Model *m,float *hyper,const int *ids,static float *step(Model *m,const int *ids,int S,int pos_base) {Cfg *c=&m->c;int H=c->hidden,W=c->hc_width,C=c->hc_count;+ m->timers.forwards++;float *hyper=falloc((int64_t)S*W);for(int s=0;s<S;s++){if(ids[s]<0||ids[s]>=c->vocab){fprintf(stderr,"token id %d outside vocabulary\n",ids[s]);exit(1);}- const float *e=m->embed+(int64_t)ids[s]*H;for(int b=0;b<C;b++)memcpy(hyper+(int64_t)s*W+(int64_t)b*H,e,(size_t)H*sizeof(float));+ float *e=hyper+(int64_t)s*W;q38_weight_row(&m->embed,ids[s],e);+ for(int b=1;b<C;b++)memcpy(e+(int64_t)b*H,e,(size_t)H*sizeof(float));}float *mixed=falloc((int64_t)S*H),*inject=falloc((int64_t)S*C),*block=falloc((int64_t)S*H);for(int i=0;i<c->layers;i++){Layer *l=&m->L[i];if(i==c->ple_layer){float *ple=falloc((int64_t)S*W);q38_ple(m,ids,S,hyper,ple);for(int64_t z=0;z<(int64_t)S*W;z++)hyper[z]+=ple[z];free(ple);}- q38_gr_read(c,&l->attn_gr,hyper,S,mixed,inject);+ q38_gr_read(m,&l->attn_gr,hyper,S,mixed,inject);if(c->is_attn[i])q38_attention(m,l,i,mixed,S,pos_base,block);else q38_deltanet(m,l,i,mixed,S,block);q38_gr_apply(c,hyper,block,inject,S);- q38_gr_read(c,&l->mlp_gr,hyper,S,mixed,inject);q38_moe(m,l,i,mixed,S,block);q38_gr_apply(c,hyper,block,inject,S);+ q38_gr_read(m,&l->mlp_gr,hyper,S,mixed,inject);q38_moe(m,l,i,mixed,S,block);q38_gr_apply(c,hyper,block,inject,S);}- q38_gr_read(c,&m->final_gr,hyper,S,mixed,NULL);m->kv_len=pos_base+S;- float *logit=falloc(c->vocab);q38_matmul(logit,mixed+(int64_t)(S-1)*H,m->lm_head,1,H,c->vocab);+ q38_gr_read(m,&m->final_gr,hyper,S,mixed,NULL);m->kv_len=pos_base+S;+ float *logit=falloc(c->vocab);double phase_started=now_s();+ q38_weight_matmul(logit,mixed+(int64_t)(S-1)*H,&m->lm_head,1,H,c->vocab);+ q38_tm_add(m,Q38_TM_LM_HEAD,phase_started);free(hyper);free(mixed);free(inject);free(block);return logit;}-static void tm_report(void) {}+static int q38_tm_enabled(void) {+ const char *enabled=getenv("COLI_TIMERS");+ return enabled&&enabled[0]=='1'&&enabled[1]=='\0';+}++static void q38_tm_report_bank(const Q38Timers *timers,const char *scope) {+ if(!q38_tm_enabled())return;+ static const char *names[Q38_TM_COUNT]={+ "expert-read","fp8-expand","routed-expert","shared-expert",+ "resident-mm","deltanet","qsa-index","qsa-attn","ple","lm-head"+ };+ double per=timers->forwards?1000.0/timers->forwards:0.0;+ fprintf(stderr,"[qwen38 timers] %s: %llu forwards\n",scope,+ (unsigned long long)timers->forwards);+ for(int i=0;i<Q38_TM_COUNT;i++)+ fprintf(stderr,"[qwen38 timers] %-14s %9.3f s %9.3f ms/forward\n",+ names[i],timers->seconds[i],timers->seconds[i]*per);+ fprintf(stderr,"[qwen38 timers] architecture phases overlap resident-mm; "+ "expert-read is disk service and fp8-expand is synchronous miss work\n");+}++static void tm_report(const Model *m) {+ q38_tm_report_bank(&m->timers,"total");+}static void q38_layer_free(Layer *l) {if(!l) return;- free(l->attn_gr.norm); free(l->attn_gr.down); free(l->attn_gr.up); free(l->attn_gr.inject);- free(l->mlp_gr.norm); free(l->mlp_gr.down); free(l->mlp_gr.up); free(l->mlp_gr.inject);- free(l->router); free(l->sh_g); free(l->sh_u); free(l->sh_d); free(l->sh_gate);- free(l->q); free(l->k); free(l->v); free(l->o); free(l->qn); free(l->kn);- free(l->idx_qk); free(l->idx_qn); free(l->idx_kn);- free(l->dn_qkv); free(l->dn_z); free(l->dn_b); free(l->dn_a); free(l->dn_conv);- free(l->dn_dtbias); free(l->dn_alog); free(l->dn_norm); free(l->dn_out);- free(l->ple_key); free(l->ple_value); free(l->ple_norm_key); free(l->ple_norm_query);+ free(l->attn_gr.norm);q38_weight_free(&l->attn_gr.down);q38_weight_free(&l->attn_gr.up);q38_weight_free(&l->attn_gr.inject);+ free(l->mlp_gr.norm);q38_weight_free(&l->mlp_gr.down);q38_weight_free(&l->mlp_gr.up);q38_weight_free(&l->mlp_gr.inject);+ q38_weight_free(&l->router);q38_weight_free(&l->sh_g);q38_weight_free(&l->sh_u);q38_weight_free(&l->sh_d);free(l->sh_gate);+ q38_weight_free(&l->q);q38_weight_free(&l->k);q38_weight_free(&l->v);q38_weight_free(&l->o);free(l->qn);free(l->kn);+ q38_weight_free(&l->idx_qk);free(l->idx_qn);free(l->idx_kn);+ q38_weight_free(&l->dn_qkv);q38_weight_free(&l->dn_z);q38_weight_free(&l->dn_b);q38_weight_free(&l->dn_a);free(l->dn_conv);+ free(l->dn_dtbias);free(l->dn_alog);free(l->dn_norm);q38_weight_free(&l->dn_out);+ q38_weight_free(&l->ple_key);q38_weight_free(&l->ple_value);free(l->ple_norm_key);free(l->ple_norm_query);free(l->ple_norm_conv); free(l->ple_conv);memset(l,0,sizeof(*l));}@@ -851,16 +1122,19 @@ static void q38_model_free(Model *m) {for(int i=0;i<m->c.layers;i++) {q38_layer_free(&m->L[i]);if(m->cache) {- /* gate/up/down share one contiguous allocation in each slot. */- for(int s=0;s<m->cache[i].n;s++) free(m->cache[i].slots[s].gate);+ for(int s=0;s<m->cache[i].n;s++){+ q38_weight_free(&m->cache[i].slots[s].gate);+ q38_weight_free(&m->cache[i].slots[s].up);+ q38_weight_free(&m->cache[i].slots[s].down);+ }free(m->cache[i].slots); free(m->cache[i].by_expert);}free(m->DN_rec ? m->DN_rec[i] : NULL); free(m->DN_conv ? m->DN_conv[i] : NULL);free(m->K ? m->K[i] : NULL); free(m->V ? m->V[i] : NULL); free(m->IK ? m->IK[i] : NULL);}free(m->L); free(m->cache); free(m->DN_rec); free(m->DN_conv); free(m->K); free(m->V); free(m->IK);- free(m->embed); free(m->lm_head);- free(m->final_gr.norm); free(m->final_gr.down); free(m->final_gr.up); free(m->final_gr.inject);+ q38_weight_free(&m->embed);q38_weight_free(&m->lm_head);+ free(m->final_gr.norm);q38_weight_free(&m->final_gr.down);q38_weight_free(&m->final_gr.up);q38_weight_free(&m->final_gr.inject);free(m->ple_history); free(m->PLE_conv_state); free(m->c.is_attn); st_destroy(&m->S);memset(m,0,sizeof(*m));}diff --git a/c/resource_plan.py b/c/resource_plan.pyindex 7c257749..f671f4ee 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -24,7 +24,7 @@ EXPERT_RE = re.compile(r"(?:model\.)?layers\.(\d+)\.(?:mlp|ffn)\.experts\.(\d+)\# sidecar and self-invalidate on any change. Best-effort: any read/write failure falls# straight back to a full recompute (see analyze_model). Sits alongside .coli_usage/.coli_ssd._ANALYSIS_CACHE_NAME = ".coli_analysis.json"-_ANALYSIS_CACHE_VERSION = 3+_ANALYSIS_CACHE_VERSION = 6def _analysis_signature(shards, config_path):@@ -53,7 +53,10 @@ def _tensor_sizes(path):start, end = meta["data_offsets"]if not 0 <= start <= end <= file_size - 8 - length:raise ValueError(f"invalid tensor offsets for {name}: {path}")- yield name, end - start+ dtype = meta.get("dtype")+ if not isinstance(dtype, str):+ raise ValueError(f"invalid tensor dtype for {name}: {path}")+ yield name, end - start, dtypedef analyze_model(model):@@ -111,20 +114,25 @@ def analyze_model(model):# storage, all of them is the mount.raise OSError(error.errno,f"{error.strerror or error}: {shard}") from error- for name, size in sizes:+ for name, size, dtype in sizes:tensor_names.add(name)- contributions = expert_contributions(resolved, name, size)+ contributions = expert_contributions(resolved, name, size, dtype)if contributions:for layer, expert, byte_count in contributions:key = (layer, expert)expert_groups[key] = expert_groups.get(key, 0) + byte_countelse:- dense_bytes += resident_contribution(resolved, name, size)+ dense_bytes += resident_contribution(resolved, name, size, dtype)layer_sizes = {}for (layer, _), size in expert_groups.items():layer_sizes.setdefault(layer, []).append(size)- per_layer = {layer: int(statistics.median(sizes)) for layer, sizes in layer_sizes.items()}+ # Most families ship uniform experts and retain the historical median to+ # tolerate container padding. Qwen3.8 explicitly accepts heterogeneous+ # source dtypes; one LRU slot may hold any expert, so each layer must be+ # priced at its largest retained representation just like the C runtime.+ reducer = max if resolved.descriptor.id == "qwen38" else statistics.median+ per_layer = {layer: int(reducer(sizes)) for layer, sizes in layer_sizes.items()}per_cap_bytes = sum(per_layer.values())typical_expert_bytes = int(statistics.median(per_layer.values())) if per_layer else 0max_expert_bytes = max(per_layer.values(), default=0)@@ -1025,6 +1033,14 @@ def environment_for_plan(plan, env=None, cuda_enabled=True):isinstance(planned_cap, bool) or planned_cap < 1)):raise ValueError("Qwen3.8 RAM budget cannot hold one expert slot per loaded layer")+ if plan.get("model", {}).get("family_id") == "qwen38":+ disabled = [name for name in ("Q38_NATIVE_FP8", "Q38_NATIVE_BF16")+ if result.get(name) == "0"]+ if disabled:+ raise ValueError(+ "Qwen3.8 auto-tier requires native expert storage; unset " ++ ", ".join(disabled) ++ " or disable auto-tier and choose an explicit cache cap")if (isinstance(planned_cap, int) and not isinstance(planned_cap, bool) andplanned_cap >= 1):# Private bridge for engines whose expert capacity is an argv value.diff --git a/c/tests/test_env_defaults.py b/c/tests/test_env_defaults.pyindex 99a1e128..3a1d3b8f 100644--- a/c/tests/test_env_defaults.py+++ b/c/tests/test_env_defaults.py@@ -80,6 +80,78 @@ class EnvDefaultsTest(unittest.TestCase):self.assertNotIn(k, e)+class SiblingPlanRefusalTest(unittest.TestCase):+ """A sibling auto-tier refusal is a launcher diagnostic, never a traceback+ or a child started with an implicit over-budget cache."""++ def setUp(self):+ self.directory = tempfile.TemporaryDirectory()+ self.model = Path(self.directory.name)+ (self.model / "config.json").write_text(+ json.dumps({"model_type": "qwen4_exp_text"}), encoding="utf-8")++ def tearDown(self):+ self.directory.cleanup()++ def refusal_args(self, **overrides):+ values = dict(+ model=str(self.model), policy="quality", ram=0, ngen=8, topp=0,+ topk=0, temp=None, repin=0, ctx=0, auto_tier=True, gpu=None,+ vram=0, kv_slots=1, no_tune_profile=True, cap=None,+ no_attach=True, attach=None, api_key=None,+ )+ values.update(overrides)+ return types.SimpleNamespace(**values)++ def plan_refusal(self):+ return mock.patch(+ "resource_plan.build_plan",+ return_value={"model": {"family_id": "qwen38"}},+ ), mock.patch(+ "resource_plan.environment_for_plan",+ side_effect=ValueError(+ "Qwen3.8 RAM budget cannot hold one expert slot per loaded layer"+ ),+ )++ def test_chat_refuses_before_popen_without_traceback(self):+ build_plan, environment = self.plan_refusal()+ with mock.patch.object(coli, "need_model"), \+ mock.patch.object(coli, "engine_for", return_value="/engines/qwen38"), \+ mock.patch.object(coli, "resource_request", return_value=(0, 0, [], 0)), \+ mock.patch("resource_plan.physical_cpu_count", return_value=8), \+ mock.patch.object(coli, "banner"), build_plan, environment, \+ mock.patch.object(coli.subprocess, "Popen") as popen:+ with self.assertRaises(SystemExit) as stopped:+ coli.cmd_chat(self.refusal_args())+ message = str(stopped.exception)+ self.assertIn("invalid resource plan", message)+ self.assertIn("one expert slot", message)+ self.assertNotIn("Traceback", message)+ popen.assert_not_called()++ def test_serve_refuses_without_starting_or_leaving_pidfile(self):+ build_plan, environment = self.plan_refusal()+ pidfile = self.model / "serve.pid"+ arguments = self.refusal_args(+ port=9123, host="127.0.0.1", model_id=None,+ cluster_coordinator=None, cluster_workers=None,+ )+ with mock.patch.object(coli, "need_model"), \+ mock.patch.object(coli, "engine_for", return_value="/engines/qwen38"), \+ mock.patch.object(coli, "resource_request", return_value=(0, 0, [], 0)), \+ mock.patch("resource_plan.physical_cpu_count", return_value=8), \+ mock.patch.object(coli, "serve_pidfile", return_value=str(pidfile)), \+ mock.patch.object(coli, "banner"), build_plan, environment, \+ mock.patch("openai_server.ARCH", "glm"), \+ mock.patch("openai_server.serve") as serve:+ with self.assertRaises(SystemExit) as stopped:+ coli.cmd_serve(arguments)+ self.assertIn("invalid resource plan", str(stopped.exception))+ serve.assert_not_called()+ self.assertFalse(pidfile.exists())++class CudaAutoEnableTest(unittest.TestCase):"""Windows bare `coli chat` (no --gpu/--vram/--auto-tier) used to ALWAYS runCPU-only even on a CUDA build with a GPU present. env_for now auto-enablesdiff --git a/c/tests/test_family_registry.py b/c/tests/test_family_registry.pyindex 9be4b5d0..c8d0cb7c 100644--- a/c/tests/test_family_registry.py+++ b/c/tests/test_family_registry.py@@ -49,7 +49,7 @@ def minimax_geometry(config, context, _model_dir):return PlannerGeometry(state, 0, 0, config["num_local_experts"])-TEST_INVENTORY = lambda _name, _size, _config: ()+TEST_INVENTORY = lambda _name, _size, _config, _dtype=None: ()QWEN36_FIXTURE = FamilyDescriptor(id="qwen36",model_types=("qwen3_5_moe_text",),@@ -246,20 +246,40 @@ class FamilyRegistryTest(unittest.TestCase):"model_dir": "."})()prefix = "model.language_model.layers.7.mlp.experts.23"self.assertEqual(expert_contributions(- resolved, prefix + ".gate_proj.weight", 256), ((7, 23, 1024),))+ resolved, prefix + ".gate_proj.weight", 256, "F8_E4M3"),+ ((7, 23, 256),))self.assertEqual(expert_contributions(- resolved, prefix + ".up_proj.weight", 512), ((7, 23, 1024),))+ resolved, prefix + ".up_proj.weight", 512, "BF16"),+ ((7, 23, 512),))self.assertEqual(expert_contributions(- resolved, prefix + ".down_proj.weight", 1024), ((7, 23, 1024),))+ resolved, prefix + ".down_proj.weight", 1024, "F32"),+ ((7, 23, 1024),))self.assertEqual(expert_contributions(- resolved, prefix + ".down_proj.weight_scale_inv", 16), ((7, 23, 0),))+ resolved, prefix + ".down_proj.weight", 512, "F16"),+ ((7, 23, 1024),))+ self.assertEqual(expert_contributions(+ resolved, prefix + ".down_proj.weight_scale_inv", 4, "F32"),+ ((7, 23, 4),))+ with self.assertRaisesRegex(ValueError, "unsupported dtype/size"):+ expert_contributions(+ resolved, prefix + ".down_proj.weight_scale_inv", 16, "F32")fused = "model.layers.7.mlp.experts.gate_up_proj"- contributions = expert_contributions(resolved, fused, 512 * 2 * 32 * 8 * 2)+ fused_size = 512 * 2 * 32 * 8 * 2+ contributions = expert_contributions(resolved, fused, fused_size, "BF16")self.assertEqual(len(contributions), 512)+ self.assertEqual(contributions[23], (7, 23, 1024))+ contributions = expert_contributions(resolved, fused, fused_size, "F16")self.assertEqual(contributions[23], (7, 23, 2048))self.assertEqual(resident_contribution(- resolved, "model.language_model.layers.0.self_attn.q_proj.weight", 100), 200)+ resolved, "model.language_model.layers.0.self_attn.q_proj.weight",+ 100, "BF16"), 100)+ self.assertEqual(resident_contribution(+ resolved, "model.language_model.layers.0.self_attn.q_proj.weight",+ 100, "F16"), 200)+ self.assertEqual(resident_contribution(+ resolved, "model.language_model.layers.0.self_attn.q_norm.weight",+ 100, "BF16"), 200)self.assertEqual(resident_contribution(resolved, "model.visual.blocks.0.attn.qkv.weight", 100), 0)self.assertEqual(resident_contribution(diff --git a/c/tests/test_qwen38_metrics.c b/c/tests/test_qwen38_metrics.cnew file mode 100644index 00000000..41eec75a--- /dev/null+++ b/c/tests/test_qwen38_metrics.c@@ -0,0 +1,54 @@+/* Qwen3.8 request metrics: timer deltas, request-local hit rate, strict TPOT+ * throughput, and the shared PROF wire schema. */+#define _GNU_SOURCE+#define QWEN38_NO_MAIN+#define QWEN38_TEST_SERVE+#include "../qwen38.c"++#define CHECK(x) do { if(!(x)){ \+ fprintf(stderr,"%s:%d: check failed: %s\n",__FILE__,__LINE__,#x);return 1; \+} } while(0)++static int close_enough(double a,double b){ return fabs(a-b)<1e-12; }++int main(void){+ CHECK(q38_decode_rate(0,1.0)==0.0);+ CHECK(q38_decode_rate(1,1e-9)==0.0);+ CHECK(q38_decode_rate(2,0.5)==2.0);+ CHECK(q38_decode_rate(3,0.5)==4.0);+ CHECK(q38_decode_rate(3,0.0)==0.0);++ CHECK(q38_cache_hit_percent(0,0)==0.0);+ CHECK(q38_cache_hit_percent(3,0)==100.0);+ CHECK(q38_cache_hit_percent(0,7)==0.0);+ CHECK(q38_cache_hit_percent(3,1)==75.0);++ Q38Timers before={0},after={0};+ for(int i=0;i<Q38_TM_COUNT;i++){+ before.seconds[i]=(double)i;+ after.seconds[i]=(double)i+0.25*(i+1);+ }+ before.forwards=7;after.forwards=11;+ Q38Timers delta=q38_tm_delta(&after,&before);+ for(int i=0;i<Q38_TM_COUNT;i++)CHECK(close_enough(delta.seconds[i],0.25*(i+1)));+ CHECK(delta.forwards==4);++ Q38Timers profile={0};+ profile.seconds[Q38_TM_EXPERT_READ]=0.1;+ profile.seconds[Q38_TM_FP8_EXPAND]=0.2;+ profile.seconds[Q38_TM_ROUTED_EXPERT]=0.3;+ profile.seconds[Q38_TM_SHARED_EXPERT]=0.4;+ profile.seconds[Q38_TM_DELTANET]=0.5;+ profile.seconds[Q38_TM_QSA_INDEX]=0.6;+ profile.seconds[Q38_TM_QSA_ATTENTION]=0.7;+ profile.seconds[Q38_TM_LM_HEAD]=0.8;+ profile.forwards=4;+ char line[256];+ CHECK(q38_format_prof(line,sizeof line,2.0,7,3,&profile)>0);+ CHECK(!strcmp(line,"PROF 2.000 7 3 0.100 0.300 0.700 1.800 0.800 4\n"));+ volatile size_t short_capacity=8;+ CHECK(q38_format_prof(line,short_capacity,2.0,7,3,&profile)<0);++ puts("qwen38 metrics: request deltas, TPOT rate, PROF schema: ok");+ return 0;+}diff --git a/c/tests/test_qwen38_native_weights.c b/c/tests/test_qwen38_native_weights.cnew file mode 100644index 00000000..eaadb18f--- /dev/null+++ b/c/tests/test_qwen38_native_weights.c@@ -0,0 +1,367 @@+/* Qwen3.8 native resident/expert weight dispatch. Besides arithmetic parity,+ * this writes a real per-expert block-FP8 safetensors container and exercises+ * the production loader and capacity-one LRU in both native and expanded+ * modes. */+#define _GNU_SOURCE+#define QWEN38_NO_MAIN+#define COLI_SEGMENT_ADAPTER+#include <pthread.h>+#include "../qwen38.c"++#define CHECK(x) do { if(!(x)){ \+ fprintf(stderr,"%s:%d: check failed: %s\n",__FILE__,__LINE__,#x);return 1; \+} } while(0)++typedef struct {+ char name[192];+ const char *dtype;+ int rows,cols;+ int64_t begin,end;+ unsigned char raw[24];+ size_t raw_bytes;+ float scale;+ int is_scale;+} FixtureTensor;++static uint16_t to_bf16(float value){+ union { float f; uint32_t u; } bits={value};return (uint16_t)(bits.u>>16);+}++/* Independent formulation of quant.h's documented native-FP8 reduction:+ * F32 accumulation inside each 128-column block and F64 accumulation after+ * applying each block scale. Do not call matmul_fp8 here—the dispatch test+ * must remain capable of detecting a defect in that production primitive. */+static void reference_fp8_matmul(float *out,const float *input,+ const uint8_t *weight,const float *scales,+ int sequences,int inputs,int outputs){+ int64_t input_blocks=fp8_nblk(inputs);+ for(int output=0;output<outputs;output++){+ for(int sequence=0;sequence<sequences;sequence++){+ double total=0.0;+ for(int64_t block=0;block<input_blocks;block++){+ int first=(int)(block*FP8_BLOCK);+ int last=first+FP8_BLOCK<inputs?first+FP8_BLOCK:inputs;+ float subtotal=0.0f;+ for(int input_index=first;input_index<last;input_index++)+ subtotal+=e4m3_decode(weight[(int64_t)output*inputs+input_index])*+ input[(int64_t)sequence*inputs+input_index];+ total+=(double)subtotal*+ (double)scales[(output/FP8_BLOCK)*input_blocks+block];+ }+ out[(int64_t)sequence*outputs+output]=(float)total;+ }+ }+}++static int add_fixture_tensor(FixtureTensor *tensors,int *count,int64_t *offset,+ const char *name,const char *dtype,int rows,int cols,+ int expert,int projection,int is_scale){+ if(*count>=12)return -1;+ FixtureTensor *tensor=&tensors[(*count)++];memset(tensor,0,sizeof(*tensor));+ int length=snprintf(tensor->name,sizeof tensor->name,"%s",name);+ if(length<0||(size_t)length>=sizeof tensor->name)return -1;+ tensor->dtype=dtype;tensor->rows=rows;tensor->cols=cols;+ tensor->begin=*offset;tensor->is_scale=is_scale;+ if(is_scale){+ tensor->scale=0.125f*(float)(1+expert*3+projection);+ *offset+=(int64_t)sizeof(float);+ }else if(!strcmp(dtype,"BF16")){+ tensor->raw_bytes=(size_t)rows*cols*sizeof(uint16_t);+ for(int index=0;index<rows*cols;index++){+ uint16_t value=to_bf16(0.0625f*(float)(1+expert*7+projection*3+index));+ memcpy(tensor->raw+(size_t)index*sizeof(value),&value,sizeof(value));+ }+ *offset+=(int64_t)tensor->raw_bytes;+ }else{+ tensor->raw_bytes=(size_t)rows*cols;+ for(int index=0;index<rows*cols;index++)+ tensor->raw[index]=(unsigned char)(0x20+expert*0x18+projection*7+index);+ *offset+=(int64_t)tensor->raw_bytes;+ }+ tensor->end=*offset;return 0;+}++static int write_fp8_fixture(const char *directory,int mixed){+ FixtureTensor tensors[12];int count=0;int64_t offset=0;+ const char *projection[3]={"gate_proj","up_proj","down_proj"};+ for(int expert=0;expert<2;expert++)for(int kind=0;kind<3;kind++){+ int rows=kind==2?2:3,cols=kind==2?3:2;char name[192];+ int length=snprintf(name,sizeof name,+ "model.layers.0.mlp.experts.%d.%s.weight",expert,projection[kind]);+ const char *dtype=mixed&&expert==1&&kind==0?"BF16":"F8_E4M3";+ if(length<0||(size_t)length>=sizeof name||+ add_fixture_tensor(tensors,&count,&offset,name,dtype,rows,cols,+ expert,kind,0))return -1;+ if(!strcmp(dtype,"BF16"))continue;+ length=snprintf(name,sizeof name,+ "model.layers.0.mlp.experts.%d.%s.weight_scale_inv",expert,+ projection[kind]);+ if(length<0||(size_t)length>=sizeof name||+ add_fixture_tensor(tensors,&count,&offset,name,"F32",1,1,+ expert,kind,1))return -1;+ }++ char header[8192];size_t used=0;header[used++]='{';+ for(int index=0;index<count;index++){+ FixtureTensor *tensor=&tensors[index];+ int length=snprintf(header+used,sizeof header-used,+ "%s\"%s\":{\"dtype\":\"%s\",\"shape\":[%d,%d],"+ "\"data_offsets\":[%lld,%lld]}",index?",":"",tensor->name,+ tensor->dtype,tensor->rows,tensor->cols,(long long)tensor->begin,+ (long long)tensor->end);+ if(length<0||(size_t)length>=sizeof header-used)return -1;+ used+=(size_t)length;+ }+ if(used+1>=sizeof header)return -1;header[used++]='}';++ char path[512];int path_length=snprintf(path,sizeof path,+ "%s/model.safetensors",directory);+ if(path_length<0||(size_t)path_length>=sizeof path)return -1;+ FILE *file=fopen(path,"wb");if(!file)return -1;+ uint64_t header_length=used;+ int failed=fwrite(&header_length,sizeof header_length,1,file)!=1||+ fwrite(header,1,used,file)!=used;+ for(int index=0;index<count&&!failed;index++){+ FixtureTensor *tensor=&tensors[index];+ if(tensor->is_scale)+ failed=fwrite(&tensor->scale,sizeof tensor->scale,1,file)!=1;+ else+ failed=fwrite(tensor->raw,1,tensor->raw_bytes,file)!=tensor->raw_bytes;+ }+ if(fclose(file))failed=1;return failed?-1:0;+}++static int init_fixture_model(Model *model,const char *directory,int native_fp8){+ memset(model,0,sizeof(*model));model->c.hidden=2;model->c.inter=3;+ model->c.experts=2;model->c.layers=1;model->range_begin=0;model->range_end=1;+ model->native_fp8=native_fp8;model->native_bf16=1;+ snprintf(model->prefix,sizeof model->prefix,"model");+ st_init(&model->S,directory);+ model->cache=(LCache*)calloc(1,sizeof(*model->cache));+ if(!model->cache){st_destroy(&model->S);return -1;}+ LCache *cache=&model->cache[0];cache->cap=1;+ cache->slots=(Slot*)calloc(1,sizeof(*cache->slots));+ cache->by_expert=(int*)malloc(2*sizeof(*cache->by_expert));+ if(!cache->slots||!cache->by_expert){+ free(cache->slots);free(cache->by_expert);free(model->cache);+ st_destroy(&model->S);memset(model,0,sizeof(*model));return -1;+ }+ cache->by_expert[0]=cache->by_expert[1]=-1;return 0;+}++static void destroy_fixture_model(Model *model){+ if(model->cache){+ for(int slot=0;slot<model->cache[0].n;slot++){+ q38_weight_free(&model->cache[0].slots[slot].gate);+ q38_weight_free(&model->cache[0].slots[slot].up);+ q38_weight_free(&model->cache[0].slots[slot].down);+ }+ free(model->cache[0].slots);free(model->cache[0].by_expert);+ }+ free(model->cache);st_destroy(&model->S);memset(model,0,sizeof(*model));+}++static uint8_t fixture_fp8_byte(int expert,int projection,int index){+ return (uint8_t)(0x20+expert*0x18+projection*7+index);+}++static float fixture_fp8_scale(int expert,int projection){+ return 0.125f*(float)(1+expert*3+projection);+}++static int verify_loaded_fp8_weight(const Q38Weight *weight,int expert,+ int projection,int native_fp8){+ int rows=projection==2?2:3,cols=projection==2?3:2;+ if(!weight||weight->rows!=rows||weight->cols!=cols||+ weight->kind!=(native_fp8?Q38_WEIGHT_FP8:Q38_WEIGHT_F32))return -1;+ float scale=fixture_fp8_scale(expert,projection);+ for(int index=0;index<rows*cols;index++){+ uint8_t raw=fixture_fp8_byte(expert,projection,index);+ if(native_fp8){+ if(((const uint8_t*)weight->data)[index]!=raw||+ weight->scale_count!=1||weight->scales[0]!=scale)return -1;+ }else if(((const float*)weight->data)[index]!=e4m3_decode(raw)*scale)+ return -1;+ }+ float input[3]={0.75f,-0.5f,0.25f},got[3]={0},want[3]={0};+ for(int row=0;row<rows;row++){+ float subtotal=0.0f;+ for(int col=0;col<cols;col++){+ float value=e4m3_decode(fixture_fp8_byte(+ expert,projection,row*cols+col));+ if(!native_fp8)value*=scale;+ subtotal+=value*input[col];+ }+ want[row]=native_fp8?(float)((double)subtotal*(double)scale):subtotal;+ }+ q38_weight_matmul(got,input,weight,1,cols,rows);+ return memcmp(got,want,(size_t)rows*sizeof(float))?-1:0;+}++static int verify_loaded_fp8_slot(const Slot *slot,int expert,int native_fp8){+ return verify_loaded_fp8_weight(&slot->gate,expert,0,native_fp8)||+ verify_loaded_fp8_weight(&slot->up,expert,1,native_fp8)||+ verify_loaded_fp8_weight(&slot->down,expert,2,native_fp8)?-1:0;+}++static int check_fixture_mode(const char *directory,int native_fp8){+ Model model;if(init_fixture_model(&model,directory,native_fp8))return -1;+ int result=-1;+ uint64_t bytes_per_capacity=0;unsigned numeric_kinds=0;+ if(q38_segment_expert_layout(&model,0,1,&bytes_per_capacity,&numeric_kinds))+ goto cleanup;+ char numeric_class[96];+ q38_segment_numeric_class(numeric_class,sizeof numeric_class,numeric_kinds);+ if(native_fp8){+ if(bytes_per_capacity!=30||numeric_kinds!=Q38_EXPERT_FP8_BLOCK||+ strcmp(numeric_class,+ "qwen38/fp8-block-f32dot-f64blocksum/cpu-v1"))goto cleanup;+ }else if(bytes_per_capacity!=72||numeric_kinds!=Q38_EXPERT_FP8_EXPANDED||+ strcmp(numeric_class,+ "qwen38/fp8-expanded-f32dot/cpu-v1"))goto cleanup;+ int capacity=0;+ if(q38_segment_cache_capacity(bytes_per_capacity,2,bytes_per_capacity*2,+ &capacity)||capacity!=2||+ !q38_segment_cache_capacity(bytes_per_capacity,2,bytes_per_capacity-1,+ &capacity))goto cleanup;+ if(q38_segment_cache_resize(&model,2)||model.cache[0].cap!=2||+ q38_segment_cache_resize(&model,1)||model.cache[0].cap!=1)goto cleanup;++ Slot *first=q38_expert_get(&model,0,0);if(!first)goto cleanup;+ Q38WeightKind expected=native_fp8?Q38_WEIGHT_FP8:Q38_WEIGHT_F32;+ if(first->gate.kind!=expected||first->up.kind!=expected||+ first->down.kind!=expected||model.miss!=1||model.hits!=0||+ verify_loaded_fp8_slot(first,0,native_fp8))goto cleanup;+ float first_value=native_fp8?e4m3_decode(((uint8_t*)first->gate.data)[0])*+ first->gate.scales[0]:+ ((float*)first->gate.data)[0];+ if(q38_expert_get(&model,0,0)!=first||model.miss!=1||model.hits!=1)goto cleanup;+ Slot *second=q38_expert_get(&model,0,1);+ if(second!=first||model.cache[0].by_expert[0]!=-1||+ model.cache[0].by_expert[1]!=0||model.miss!=2||model.hits!=1||+ verify_loaded_fp8_slot(second,1,native_fp8))goto cleanup;+ float second_value=native_fp8?e4m3_decode(((uint8_t*)second->gate.data)[0])*+ second->gate.scales[0]:+ ((float*)second->gate.data)[0];+ if(first_value==second_value)goto cleanup;+ if(q38_expert_get(&model,0,0)!=first||model.cache[0].by_expert[0]!=0||+ model.cache[0].by_expert[1]!=-1||model.miss!=3)goto cleanup;+ result=0;+cleanup:+ destroy_fixture_model(&model);return result;+}++static int check_mixed_layout(const char *directory,int native_fp8,+ uint64_t expected_bytes,unsigned expected_kinds,+ const char *expected_class){+ Model model;if(init_fixture_model(&model,directory,native_fp8))return -1;+ uint64_t bytes=0;unsigned kinds=0;char numeric_class[96];int result=-1;+ if(!q38_segment_expert_layout(&model,0,1,&bytes,&kinds)){+ q38_segment_numeric_class(numeric_class,sizeof numeric_class,kinds);+ if(bytes==expected_bytes&&kinds==expected_kinds&&+ !strcmp(numeric_class,expected_class)){+ Slot *slot=q38_expert_get(&model,0,0);+ if(slot&&!verify_loaded_fp8_slot(slot,0,native_fp8)&&+ q38_expert_get(&model,0,1)==slot&&+ slot->gate.kind==Q38_WEIGHT_BF16&&+ slot->up.kind==(native_fp8?Q38_WEIGHT_FP8:Q38_WEIGHT_F32)&&+ slot->down.kind==(native_fp8?Q38_WEIGHT_FP8:Q38_WEIGHT_F32)&&+ model.cache[0].by_expert[0]==-1&&+ model.cache[0].by_expert[1]==0){+ int bf16_ok=1;+ const uint16_t *gate=(const uint16_t*)slot->gate.data;+ for(int index=0;index<6;index++)+ if(gate[index]!=to_bf16(0.0625f*(float)(8+index)))bf16_ok=0;+ if(bf16_ok&&!verify_loaded_fp8_weight(&slot->up,1,1,native_fp8)&&+ !verify_loaded_fp8_weight(&slot->down,1,2,native_fp8)&&+ q38_expert_get(&model,0,0)==slot&&+ slot->gate.kind==(native_fp8?Q38_WEIGHT_FP8:Q38_WEIGHT_F32)&&+ model.cache[0].by_expert[0]==0&&+ model.cache[0].by_expert[1]==-1)result=0;+ }+ }+ }+ destroy_fixture_model(&model);return result;+}++static int check_segment_failure_outputs(void){+ char error[128];void *output=(void*)(uintptr_t)1;+ ColiSegmentCapabilities capabilities;+ ColiSegmentEngineOptions engine_options={+ .struct_size=sizeof(engine_options),.model_dir="unused",+ .context_tokens=1,.backend_mask=COLI_SEGMENT_CAP_CUDA,+ };+ if(!qwen38_segment_engine_open(&output,&capabilities,&engine_options,+ error,sizeof error)||output)return -1;++ Qwen38SegmentEngine engine={0};engine.context_tokens=1;+ ColiSegmentSessionOptions session_options={+ .struct_size=sizeof(session_options),.context_tokens=2,+ };+ output=(void*)(uintptr_t)1;+ if(!qwen38_segment_session_create(&engine,&output,&session_options,+ error,sizeof error)||output)return -1;+ return 0;+}++int main(void){+ enum { S=2, I=257, O=129 };+ Q38Weight fp8={0};q38_weight_reserve(&fp8,Q38_WEIGHT_FP8,O,I);+ CHECK(fp8.scale_count==6);CHECK(q38_weight_bytes(&fp8)==(uint64_t)O*I+6*sizeof(float));+ uint8_t *raw=(uint8_t*)fp8.data;+ for(int64_t i=0;i<fp8.elements;i++)raw[i]=(uint8_t)((i*17+13)%0x7f);+ for(int64_t block=0;block<fp8.scale_count;block++)+ fp8.scales[block]=(float)(block+1)*0.25f*(block&1?-1.0f:1.0f);+ float x[S*I];for(int i=0;i<S*I;i++)x[i]=(float)((i%19)-9)/7.f;+ float want[S*O],got[S*O];+ reference_fp8_matmul(want,x,raw,fp8.scales,S,I,O);+ q38_weight_matmul(got,x,&fp8,S,I,O);+ CHECK(!memcmp(want,got,sizeof want));+ void *same=fp8.data;q38_weight_reserve(&fp8,Q38_WEIGHT_FP8,O,I);+ CHECK(fp8.data==same);+ q38_weight_free(&fp8);CHECK(fp8.kind==Q38_WEIGHT_NONE&&!fp8.data&&!fp8.scales);++ enum { BI=9, BO=4, BS=3 };+ Q38Weight bf16={0};q38_weight_reserve(&bf16,Q38_WEIGHT_BF16,BO,BI);+ uint16_t *half=(uint16_t*)bf16.data;float full[BO*BI];+ for(int i=0;i<BO*BI;i++){+ float value=(float)((i%11)-5)*0.125f;+ half[i]=to_bf16(value);full[i]=bf16_to_f32(half[i]);+ }+ float bx[BS*BI],bwant[BS*BO],bgot[BS*BO];+ for(int i=0;i<BS*BI;i++)bx[i]=(float)((i%7)-3)*0.25f;+ q38_matmul(bwant,bx,full,BS,BI,BO);+ q38_weight_matmul(bgot,bx,&bf16,BS,BI,BO);+ CHECK(!memcmp(bwant,bgot,sizeof bwant));+ float row[BI];q38_weight_row(&bf16,2,row);+ CHECK(!memcmp(row,full+2*BI,sizeof row));+ CHECK(q38_weight_bytes(&bf16)==(uint64_t)BO*BI*sizeof(uint16_t));+ q38_weight_free(&bf16);++ char directory[]="test_qwen38_native_XXXXXX";CHECK(mkdtemp(directory)!=NULL);+ CHECK(write_fp8_fixture(directory,0)==0);+ CHECK(check_fixture_mode(directory,1)==0);+ CHECK(check_fixture_mode(directory,0)==0);+ char path[512];int path_length=snprintf(path,sizeof path,+ "%s/model.safetensors",directory);+ CHECK(path_length>0&&(size_t)path_length<sizeof path);+ CHECK(remove(path)==0);CHECK(rmdir(directory)==0);++ char mixed_directory[]="test_qwen38_mixed_XXXXXX";+ CHECK(mkdtemp(mixed_directory)!=NULL);+ CHECK(write_fp8_fixture(mixed_directory,1)==0);+ CHECK(check_mixed_layout(mixed_directory,1,32,+ Q38_EXPERT_FP8_BLOCK|Q38_EXPERT_BF16,+ "qwen38/mixed-expert-arithmetic-05/cpu-v1")==0);+ CHECK(check_mixed_layout(mixed_directory,0,72,+ Q38_EXPERT_FP8_EXPANDED|Q38_EXPERT_BF16,+ "qwen38/mixed-expert-arithmetic-06/cpu-v1")==0);+ path_length=snprintf(path,sizeof path,"%s/model.safetensors",mixed_directory);+ CHECK(path_length>0&&(size_t)path_length<sizeof path);+ CHECK(remove(path)==0);CHECK(rmdir(mixed_directory)==0);++ CHECK(check_segment_failure_outputs()==0);++ puts("qwen38 native weights: independent FP8 oracle, real loader, LRU, mixed classes, adapter failures: ok");+ return 0;+}diff --git a/c/tests/test_qwen38_tokenizer.c b/c/tests/test_qwen38_tokenizer.cindex 68cd0b9d..393e50f1 100644--- a/c/tests/test_qwen38_tokenizer.c+++ b/c/tests/test_qwen38_tokenizer.c@@ -159,10 +159,10 @@ int main(void) {uint8_t layer_kinds[2]={0,1};Cfg cfg={0};cfg.layers=2;cfg.is_attn=layer_kinds;cfg.hidden=10;cfg.inter=2;cfg.experts=9;- int cache_capacity=0;uint64_t slot_bytes=0;- CHECK(q38_segment_cache_capacity(&cfg,2,960,&cache_capacity,&slot_bytes)==0);- CHECK(slot_bytes==240&&cache_capacity==2);- CHECK(q38_segment_cache_capacity(&cfg,2,479,&cache_capacity,&slot_bytes)<0);+ int cache_capacity=0;+ CHECK(q38_segment_cache_capacity(480,cfg.experts,960,&cache_capacity)==0);+ CHECK(cache_capacity==2);+ CHECK(q38_segment_cache_capacity(480,cfg.experts,479,&cache_capacity)<0);cfg.dn_vheads=2;cfg.dn_kdim=3;cfg.dn_vdim=4;cfg.dn_conv_dim=5;cfg.dn_convk=3;cfg.kv_heads=2;cfg.head_dim=4;cfg.idx_dim=3;cfg.ple_layer=0;cfg.hc_width=8;cfg.ple_convk=3;cfg.ngram_size=3;diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex 08903c3a..bed759cb 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -27,8 +27,11 @@ def write_shard(path, tensors):offset = 0header = {}payload = b""- for name, size in tensors:- header[name] = {"dtype": "U8", "shape": [size], "data_offsets": [offset, offset + size]}+ for tensor in tensors:+ name, size, *metadata = tensor+ dtype = metadata[0] if metadata else "U8"+ header[name] = {"dtype": dtype, "shape": [size],+ "data_offsets": [offset, offset + size]}payload += b"\0" * sizeoffset += sizeraw = json.dumps(header).encode()@@ -638,7 +641,7 @@ memInfo.free: 23.50 GB (97%)self.assertEqual(plan["tiers"]["vram"]["devices"], [])self.assertIn("not detected", plan["warnings"][0])- def test_qwen38_cpu_only_plan_ignores_inventory_and_exports_cache_cap(self):+ def test_qwen38_cpu_only_plan_prices_heterogeneous_cache_and_exports_cap(self):config = {"model_type": "qwen4_exp","text_config": {@@ -663,13 +666,22 @@ memInfo.free: 23.50 GB (97%)},}(self.model / "config.json").write_text(json.dumps(config))- tensors = [("model.embed_tokens.weight", 128)]- for expert in range(2):- for projection in ("gate_proj", "up_proj", "down_proj"):- tensors.append((- f"model.layers.0.mlp.experts.{expert}.{projection}.weight", 32- ))+ tensors = [("model.embed_tokens.weight", 256, "BF16")]+ for projection in ("gate_proj", "up_proj", "down_proj"):+ prefix = f"model.layers.0.mlp.experts.0.{projection}"+ tensors.append((prefix + ".weight", 32, "F8_E4M3"))+ tensors.append((prefix + ".weight_scale_inv", 4, "F32"))+ tensors.append((+ f"model.layers.0.mlp.experts.1.{projection}.weight", 128, "F32"+ ))write_shard(self.model / "model.safetensors", tensors)+ analysis = analyze_model(self.model)+ self.assertEqual(analysis["dense_bytes"], 256)+ self.assertEqual(analysis["expert_bytes"], 492)+ # A slot can receive either expert. It must use the larger retained+ # representation, not the median of unlike source dtypes.+ self.assertEqual(analysis["expert_bytes_by_layer"], {0: 384})+ self.assertEqual(analysis["per_cap_bytes"], 384)gpu = {"index": 0, "name": "unrelated", "total_bytes": 16 * GB,"free_bytes": 14 * GB, "unified_memory": True}plan = build_plan(self.model, context=64, available_memory=16 * GB,@@ -680,6 +692,10 @@ memInfo.free: 23.50 GB (97%)cap = plan["tiers"]["ram"]["cache_slots_per_layer"]self.assertGreaterEqual(cap, 1)self.assertEqual(environment_for_plan(plan)["COLI_PLAN_CAP"], str(cap))+ for variable in ("Q38_NATIVE_FP8", "Q38_NATIVE_BF16"):+ with self.subTest(variable=variable), self.assertRaisesRegex(+ ValueError, "requires native expert storage"):+ environment_for_plan(plan, {variable: "0"})plan["tiers"]["ram"]["cache_slots_per_layer"] = 0with self.assertRaisesRegex(ValueError, "one expert slot"):environment_for_plan(plan)diff --git a/c/tests/test_segment_adapters_real.c b/c/tests/test_segment_adapters_real.cindex c392dc7a..aa4da1b8 100644--- a/c/tests/test_segment_adapters_real.c+++ b/c/tests/test_segment_adapters_real.c@@ -206,6 +206,13 @@ int main(int argc, char **argv) {snprintf(error, sizeof(error), "incompatible real-adapter capabilities");goto cleanup;}+ if(!strcmp(family,"qwen38")&&+ (strcmp(full_caps.numeric_class,"qwen38/bf16-values-f32dot/cpu-v1")||+ strcmp(full_caps.numeric_class,left_caps.numeric_class)||+ strcmp(full_caps.numeric_class,right_caps.numeric_class))){+ snprintf(error,sizeof(error),"Qwen3.8 reported an inaccurate numeric class: %s",+ full_caps.numeric_class);goto cleanup;+ }if (create_session(full_engine, context, &full, error, sizeof(error)) ||create_session(full_engine, context, &restored, error, sizeof(error)) ||create_session(left_engine, context, &left, error, sizeof(error)) ||diff --git a/c/tools/make_qwen38_oracle.py b/c/tools/make_qwen38_oracle.pyindex da95d918..cf3d5dcf 100644--- a/c/tools/make_qwen38_oracle.py+++ b/c/tools/make_qwen38_oracle.py@@ -3,9 +3,9 @@This is intended for the text-only fixture produced by ``make_qwen38_tiny.py``and records both a cache prefill and a one-token cached decode. It also emits-greedy generated IDs and SHA-256 digests of logits/hidden states, which avoids-putting large floating-point arrays in JSON while still detecting numerical-or tensor-layout changes.+greedy generated IDs, SHA-256 digests of logits/hidden states, and the final+prediction logits used by the C numeric gate. Earlier intermediate tensors stay+as digests so the JSON remains compact while still detecting layout changes.The released Qwen3.8 checkpoint is multimodal and includes vision/MTP tensors;this script intentionally refuses a multimodal config rather than silentlydiff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex d26c7fcc..05464a1c 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -16,7 +16,7 @@ what follows, but the sister engines read their own:| `kimi_k3` | `c/kimi_k3.c` | the `K3_*` family — see [Kimi K3 engine](#kimi-k3-engine-kimi_k3) || `inkling` | `c/inkling.c` | `INK_*`, plus `CTX_MAX`, `PIN_N`, `REP_PEN`, `GPU_DEV`, `NOGPU` — see [Inkling engine](#inkling-engine-inkling) || `qwen36` | `c/qwen36.c` | `QWEN_*`, `Q36_*`, and its dense/CUDA-tier controls — see [Qwen3.6 engine](#qwen36-engine-qwen36) |-| `qwen38` | `c/qwen38.c` | `Q38_MAXT`, `Q38_EOS` — see [Qwen3.8 engine](#qwen38-engine-qwen38) |+| `qwen38` | `c/qwen38.c` | `Q38_MAXT`, `Q38_EOS`, `Q38_NATIVE_FP8`, `Q38_NATIVE_BF16`, `COLI_TIMERS` — see [Qwen3.8 engine](#qwen38-engine-qwen38) || `olmoe` | `c/olmoe.c` | `HOT`, `WIDE`, `SMOOTH`, `CONF_LIMIT`, `MAX_NEW`, `CHAT`, `EXPERT_DROP`, `WARMUP` — see [OLMoE engine](#olmoe-engine-olmoe) || `deepseek_v4` | `c/deepseek_v4.c` | `CTX`, the `V4_*` / `DSV4_*` families and the two `COLI_CUDA_*_BATCH` gates — see [DeepSeek V4 engine](#deepseek-v4-engine-deepseek_v4); note that the CUDA section below describes `colibri.c` knobs (`COLI_CUDA`, `CUDA_DENSE`, ...) which the V4 engine does not read — its GPU switch is `DSV4_CUDA` |@@ -381,6 +381,9 @@ checkpoint layout and the text-only capability boundary.|---|---|---|| `Q38_MAXT` | `8192` | Served context capacity. Values above the model's native 262,144-token limit are clamped; malformed or non-positive values restore the default. || `Q38_EOS` | tokenizer/config stop IDs | Override the served end-of-sequence token ID for controlled experiments. Normally the engine stops on the tokenizer's `<|im_end|>` / `<|endoftext|>` IDs, falling back to `eos_token_id`. |+| `Q38_NATIVE_FP8` | `1` (on) | Keep routed E4M3 expert bytes and their F32 128×128 block scales native in the LRU. `=0` restores expanded-FP32 slots for A/B validation. |+| `Q38_NATIVE_BF16` | `1` (on) | Keep resident and routed BF16 matrices in two-byte storage while retaining FP32 activations/accumulation. `=0` restores the expanded-FP32 reference. |+| `COLI_TIMERS` | `0` (off) | Set to `1` for the detailed Qwen3.8 phase breakdown on stderr. The shared per-request `PROF` frame is emitted regardless. |## DeepSeek V4 engine (`deepseek_v4`)diff --git a/docs/qwen38.md b/docs/qwen38.mdindex 21e4c81c..77ecf40b 100644--- a/docs/qwen38.md+++ b/docs/qwen38.md@@ -33,20 +33,24 @@ by default; `reasoning_effort` accepts `low`, `medium`, `high`, and `xhigh`, andTools, image content, audio, and grammar constraints are rejected explicitly.The official FP8 repository is about 185.5 GB in decimal units (roughly 173-GiB). The default CPU engine expands its BF16 resident text tensors to f32 and-keeps selected FP8 experts in a bounded per-layer cache; 32 GB of RAM is a-practical starting point at the default 8,192-token context. Context state grows-by about 54 KiB per token. There is currently no Qwen3.8 GPU backend.+GiB). The default CPU engine keeps resident BF16 matrices in their native+two-byte representation and selected experts in native block-FP8. Activations+are FP32; BF16 matrices use FP32 accumulation, while native FP8 uses FP32 dot+products within each 128-column block and FP64 accumulation across the scaled+blocks. The bounded per-layer cache therefore spends about one quarter of the+previous memory per FP8 expert. Context state grows by about 54 KiB per token.+There is currently no Qwen3.8 GPU backend.## What stays on diskThe 51B-parameter PLE table is never materialized in RAM. For each token theengine hashes its bigram and trigram history, reads sixteen 160-byte FP8 rows,and applies the checkpoint's scalar PLE scale. Routed experts are likewise read-on demand: the official per-expert E4M3 gate, up, and down matrices are decoded-with their 128 x 128 `weight_scale_inv` blocks into an LRU whose capacity is the-first engine positional argument. The launcher defaults to one expert slot per-layer.+on demand: the official per-expert E4M3 gate, up, and down bytes and their+128 x 128 floating-point `weight_scale_inv` blocks remain native in an LRU whose+capacity is the first engine positional argument. The canonical FP8 matmul+decodes values during accumulation rather than materializing three FP32 matrices+per slot. The launcher defaults to one expert slot per layer.QSA caches the two K/V heads and the indexer's raw key. Complete four-tokenblocks are pooled and scored, the best 512 blocks are retained, and a causal@@ -55,6 +59,23 @@ only on those original tokens. The native model limit is 262,144 tokens;`Q38_MAXT` defaults to 8,192 and may raise the server limit up to that nativeceiling when the required RAM is available.+## Performance telemetry++Every served request reports its own routed-expert cache hit rate; persistent+engine counters are differenced at request boundaries rather than exposed as a+cumulative percentage. `DONE` decode throughput is based on completed+inter-token intervals: the first token comes from prefill, so a one-token reply+correctly reports zero decode tok/s instead of an artificial near-infinite rate.+The request wall time, prompt/completion counts, expert read/wait/matmul time,+sequence-mixer time, LM-head time and actual forward count are emitted through+the shared `PROF` frame consumed by `tools/datapoint.py`.++Set `COLI_TIMERS=1` for the finer Qwen-specific breakdown on stderr: expert+reads, FP8 expansion, routed and shared experts, resident matmuls, DeltaNet, QSA+indexing, QSA attention, PLE and the LM head. Architecture-phase times overlap+the resident-matmul counter by design; expert disk service also overlaps the+synchronous miss-wait value in `PROF`.+## Correctness gateThe tiny oracle is generated from the `Qwen4ExpForCausalLM` class in@@ -69,8 +90,9 @@ make -C c qwen38-tiny-checkIt exercises Gated DeltaNet, sparse QSA above its token budget, PLE, four gatedresidual streams, routed and shared experts, prefill, cached decode, and LRUeviction. The gate checks both greedy token IDs and the final upstream logit-vector. CI repeats the capacity-one path under ASan and UBSan and verifies that-a config/tensor shape disagreement is refused.+vector, and runs both native-BF16 and expanded-FP32 resident modes at cache+capacities one and four. CI repeats the capacity-one path under ASan and UBSan+and verifies that a config/tensor shape disagreement is refused.## Supported checkpoint layouts@@ -78,7 +100,10 @@ The released multimodal checkpoint stores text tensors below`model.language_model`; the standalone upstream text class stores them below`model`. Both prefixes are accepted. Routed experts may be the officialper-expert block-FP8 matrices or the fused BF16 tensors emitted by the upstream-text class. Other model types and incompatible tensor shapes fail during load.+text class. `Q38_NATIVE_FP8=0` restores the former expanded-FP32 expert cache for+numerical/performance A/Bs; `Q38_NATIVE_BF16=0` does the same for resident and+routed BF16 matrices. Other model types, unsupported scale encodings, and+incompatible tensor shapes fail during load.The weights remain covered by the Qwen Community License 1.0 in the downloadedcheckpoint. They are not redistributed by Colibri.