Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
feat(k3): map prepared weights read-only
bherald6e5f36a1657b
diff --git a/c/Makefile b/c/Makefileindex a24cd227..f216e4b6 100644--- a/c/Makefile+++ b/c/Makefile@@ -763,6 +763,9 @@ tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicotests/test_k3_ram_budget$(EXE): tests/test_k3_ram_budget.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_k3_mmap$(EXE): tests/test_k3_mmap.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h+ $(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)+tests/test_tok_kimi_tiny$(EXE): tests/test_tok_kimi_tiny.c tok.h tok_unicode.h tok_unicode_o200k.h json.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.hdiff --git a/c/kimi_k3.c b/c/kimi_k3.cindex 153e1a99..3bca2ab7 100644--- a/c/kimi_k3.c+++ b/c/kimi_k3.c@@ -39,6 +39,8 @@* K3_BITS=4|8|32 load-time quant of KDA/latent/shared/dense (default 4)* K3_MLA_BITS=8|4|32 MLA projections (default 8)* K3_HEAD_BITS=8|4|32 lm_head (default 8)+ * K3_MMAP=0|1 map prepared U8 + F32 tensors read-only instead of+ * copying them to private heap (default 0; CPU-only)* K3_EXPERT_GB=N routed-expert LRU cache budget (default 8)* K3_VK=0|1 Vulkan tier (build with `make VK=1 kimi_k3`; default* 1 when built): shared experts VRAM-resident + a@@ -110,9 +112,11 @@ typedef struct {int bos, eos[8], n_eos;} Cfg;-/* ---------- RAM-resident weight, quantized at load ---------- */+/* ---------- RAM-resident or read-only mapped weight ---------- */typedef struct { int fmt; float *f; int8_t *q8; uint8_t *q4; float *s; int O, I, gs;- void *vk; /* ColiVkTensor* once uploaded (K3_VK); NULL = CPU only */ } W;+ void *vk; /* ColiVkTensor* once uploaded (K3_VK); NULL = CPU only */+ st_mapped_raw data_map, scale_map;+ int mapped; } W;typedef struct { /* KDA layer */W q, k, v, o, g;@@ -335,6 +339,8 @@ static float w_rowdot(const W *w, int r, const float *x){#define QCHUNK 1024 /* rows per load-quantize pass */static int g_bits_env=0; /* K3_BITS explicitly set: enables the* int8-container -> int4 load downcast */+static int g_k3_mmap=0; /* prepared U8/F32 tensors stay file-backed */+static uint64_t g_k3_mmap_bytes=0, g_k3_mmap_views=0;#ifdef COLI_CUDAstatic int g_k3_cuda=0; /* K3_CUDA=1: MXFP4 routed experts on CUDA at decode */#endif@@ -342,6 +348,41 @@ static int g_k3_direct=-1; /* K3_DIRECT: O_DIRECT expert reads */static int g_k3_idot=1; /* K3_IDOT: int8-activation expert matmuls */static int g_k3_pipe=1; /* K3_PIPE: overlap loads with compute */static float g_k3_topp=0.f; /* K3_TOPP: routed-expert top-p pruning */++static int k3_mmap_backend_allowed(int mmap_enabled, int vk_enabled, int cuda_enabled){+ return !mmap_enabled || (!vk_enabled && !cuda_enabled);+}++static void w_map_prepared(Model *m, W *w, const char *name, const char *scale_name,+ st_tensor *t, st_tensor *ts){+ if(ts->dtype!=2){+ fprintf(stderr,"K3_MMAP=1 requires F32 scale sidecars; %s is %s -- refusing\n",+ scale_name,st_dtype_name(ts->dtype)); exit(1);+ }+ if(st_map_raw(&m->S,name,&w->data_map)!=0){+ fprintf(stderr,"K3_MMAP=1 could not map %s: %s -- refusing\n",name,strerror(errno)); exit(1);+ }+ if(st_map_raw(&m->S,scale_name,&w->scale_map)!=0){+ int saved=errno; st_unmap_raw(&w->data_map);+ fprintf(stderr,"K3_MMAP=1 could not map %s: %s -- refusing\n",scale_name,strerror(saved)); exit(1);+ }+ if(((uintptr_t)w->scale_map.data%_Alignof(float))!=0){+ st_unmap_raw(&w->scale_map); st_unmap_raw(&w->data_map);+ fprintf(stderr,"K3_MMAP=1 scale sidecar %s is not float-aligned -- refusing\n",scale_name); exit(1);+ }+ w->mapped=1;+ w->s=(float*)w->scale_map.data;+ g_k3_mmap_bytes+=(uint64_t)t->nbytes+(uint64_t)ts->nbytes;+ g_k3_mmap_views+=2;+}++static void w_release_host(W *w){+ if(!w) return;+ if(w->mapped){ st_unmap_raw(&w->scale_map); st_unmap_raw(&w->data_map); }+ else { free(w->f); free(w->q8); free(w->q4); free(w->s); }+ memset(w,0,sizeof(*w));+}+static void w_load(Model *m, W *w, const char *name, int O, int I, int bits){char nm[512]; snprintf(nm,sizeof(nm),"%s%s",m->pfx,name);st_tensor *t=st_find(&m->S,nm);@@ -356,6 +397,10 @@ static void w_load(Model *m, W *w, const char *name, int O, int I, int bits){if(!ts){ fprintf(stderr,"%s: quantized (U8) but no %s scale sidecar\n",nm,qn); exit(1); }if(t->nbytes==(int64_t)O*I && ts->numel==O){ /* int8 per-row */if(g_bits_env && bits==4 && I%64==0){+ if(g_k3_mmap){+ fprintf(stderr,"K3_MMAP=1 cannot combine with load-time int8-to-int4 conversion for %s; use a prepared int4 container or unset K3_BITS -- refusing\n",nm);+ exit(1);+ }/* EXPLICIT K3_BITS=4 on an int8 container: downcast to int4-g64* at load. Halves resident RAM (the 62 GB box cannot hold the* 93-layer non-expert set at int8 next to a desktop session);@@ -382,12 +427,21 @@ static void w_load(Model *m, W *w, const char *name, int O, int I, int bits){free(q8); free(s8);return;}- w->fmt=1; w->q8=malloc((size_t)O*I);+ w->fmt=1;+ if(g_k3_mmap){+ w_map_prepared(m,w,nm,qn,t,ts); w->q8=(int8_t*)w->data_map.data;+ return;+ }+ w->q8=malloc((size_t)O*I);if(!w->q8){fprintf(stderr,"OOM int8 %s\n",nm);exit(1);}st_read_raw(&m->S,nm,w->q8,1);w->s=falloc(O); st_read_f32(&m->S,qn,w->s,0);} else if(I%64==0 && t->nbytes==(int64_t)O*(I/2) && ts->numel==(int64_t)O*(I/64)){w->fmt=4; w->gs=64; /* int4-g64 */+ if(g_k3_mmap){+ w_map_prepared(m,w,nm,qn,t,ts); w->q4=(uint8_t*)w->data_map.data;+ return;+ }w->q4=malloc((size_t)O*(I/2));if(!w->q4){fprintf(stderr,"OOM int4 %s\n",nm);exit(1);}st_read_raw(&m->S,nm,w->q4,1);@@ -398,6 +452,10 @@ static void w_load(Model *m, W *w, const char *name, int O, int I, int bits){}return;}+ if(g_k3_mmap){+ fprintf(stderr,"K3_MMAP=1 requires a fully prepared U8 + F32-sidecar container; %s is %s and would need load-time conversion -- refusing\n",+ nm,st_dtype_name(t->dtype)); exit(1);+ }if(t->numel!=(int64_t)O*I){ fprintf(stderr,"%s: numel %lld != %dx%d\n",nm,(long long)t->numel,O,I); exit(1); }if(bits>=32){ w->fmt=0; w->f=falloc((int64_t)O*I); st_read_f32(&m->S,nm,w->f,0); return; }int gs=64;@@ -549,6 +607,21 @@ static void model_init(Model *m, const char *snap, int n_layers_env){snprintf(m->pfx,sizeof(m->pfx),"language_model.");if((c->n_layers+c->res_bs-1)/c->res_bs+1>16){ fprintf(stderr,"attn_res: too many blocks\n"); exit(1); }g_bits_env = getenv("K3_BITS")!=NULL;+ g_k3_mmap = getenv("K3_MMAP")?atoi(getenv("K3_MMAP")):0;+ if(g_k3_mmap){+ int vk_requested=0, cuda_requested=0;+#ifdef COLI_VULKAN+ { const char *ev=getenv("K3_VK"); vk_requested=!ev||atoi(ev); }+#endif+#ifdef COLI_CUDA+ cuda_requested=getenv("K3_CUDA")&&atoi(getenv("K3_CUDA"));+#endif+ if(!k3_mmap_backend_allowed(1,vk_requested,cuda_requested)){+ fprintf(stderr,"K3_MMAP=1 is CPU-only; set K3_VK=0 and K3_CUDA=0 -- refusing before model load\n");+ exit(1);+ }+ fprintf(stderr,"[K3-MMAP] prepared tensor mapping enabled (CPU-only, no conversion fallback)\n");+ }#ifdef COLI_CUDAg_k3_cuda = getenv("K3_CUDA") ? atoi(getenv("K3_CUDA")) : 0;if(g_k3_cuda){@@ -657,6 +730,9 @@ static void model_init(Model *m, const char *snap, int n_layers_env){free(rn); free(rp); }w_load(m,&m->lm_head,"lm_head.weight",c->vocab,c->hidden,hbits);} else fprintf(stderr,"[K3] final norm/lm_head not present β trace-only mode\n");+ if(g_k3_mmap)+ fprintf(stderr,"[K3-MMAP] %llu views, %.1f GB prepared weights/scales file-backed\n",+ (unsigned long long)g_k3_mmap_views,g_k3_mmap_bytes/1e9);#ifdef COLI_VULKAN{ const char *ev=getenv("K3_VK");if(!ev||atoi(ev)){diff --git a/c/tests/test_k3_mmap.c b/c/tests/test_k3_mmap.cnew file mode 100644index 00000000..a4c92da3--- /dev/null+++ b/c/tests/test_k3_mmap.c@@ -0,0 +1,79 @@+/* K3_MMAP: prepared U8 weights and F32 scales must produce exactly the same+ * CPU math as the existing heap-copy loader while remaining file-backed. */+#define main kimi_k3_main_unused+#include "../kimi_k3.c"+#undef main++#include <stdio.h>++static int failures;+#define CHECK(cond, ...) do { if(!(cond)){ \+ fprintf(stderr,"FAIL %s:%d: ",__FILE__,__LINE__); \+ fprintf(stderr,__VA_ARGS__); fputc('\n',stderr); failures++; } } while(0)++static void write_fixture(const char *dir){+ char path[512]; snprintf(path,sizeof(path),"%s/model.safetensors",dir);+ const char *hdr=+ "{\"w8\":{\"dtype\":\"U8\",\"shape\":[2,4],\"data_offsets\":[0,8]},"+ "\"w8.qs\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[8,16]},"+ "\"w4\":{\"dtype\":\"U8\",\"shape\":[2,32],\"data_offsets\":[16,80]},"+ "\"w4.qs\":{\"dtype\":\"F32\",\"shape\":[2,1],\"data_offsets\":[80,88]}}";+ const int8_t q8[8]={-7,3,9,12,5,-4,2,11};+ const float s8[2]={0.25f,1.5f};+ uint8_t q4[64]; float s4[2]={0.125f,0.75f};+ for(int i=0;i<64;i++) q4[i]=(uint8_t)((i*13+7)&255);+ size_t raw_hlen=strlen(hdr);+ size_t pad=(8-((8+raw_hlen)%8))%8;+ uint64_t hlen=(uint64_t)(raw_hlen+pad);+ FILE *f=fopen(path,"wb");+ fwrite(&hlen,8,1,f); fwrite(hdr,1,raw_hlen,f);+ for(size_t i=0;i<pad;i++) fputc(' ',f);+ fwrite(q8,1,sizeof(q8),f); fwrite(s8,1,sizeof(s8),f);+ fwrite(q4,1,sizeof(q4),f); fwrite(s4,1,sizeof(s4),f); fclose(f);+}++static void compare_weight(Model *m, const char *name, int O, int I, int bits){+ W copied={0}, mapped={0}; float x[64], yc[2]={0}, ym[2]={0};+ float ac[64]={0}, am[64]={0};+ for(int i=0;i<I;i++) x[i]=(float)(i%9-4)*0.125f;++ g_k3_mmap=0; g_bits_env=0; w_load(m,&copied,name,O,I,bits);+ g_k3_mmap=1; w_load(m,&mapped,name,O,I,bits);+ CHECK(mapped.mapped,"%s did not use mapped storage",name);+ CHECK(!copied.mapped,"%s changed the default loader",name);+ CHECK(mapped.s==(float*)mapped.scale_map.data,"%s scales are not mapped",name);+ if(mapped.fmt==1) CHECK(mapped.q8==(int8_t*)mapped.data_map.data,"%s int8 data are not mapped",name);+ if(mapped.fmt==4) CHECK(mapped.q4==(uint8_t*)mapped.data_map.data,"%s int4 data are not mapped",name);++ w_matmul(yc,x,&copied,1); w_matmul(ym,x,&mapped,1);+ CHECK(!memcmp(yc,ym,sizeof(yc)),"%s matmul output differs",name);+ w_addrow(&copied,1,0.75f,ac); w_addrow(&mapped,1,0.75f,am);+ CHECK(!memcmp(ac,am,(size_t)I*sizeof(float)),"%s row expansion differs",name);+ float dc=w_rowdot(&copied,1,x), dm=w_rowdot(&mapped,1,x);+ CHECK(!memcmp(&dc,&dm,sizeof(dc)),"%s row dot differs",name);++ w_release_host(&mapped); w_release_host(&copied);+}++int main(void){+ char dir[]="test_k3_mmap_XXXXXX"; CHECK(mkdtemp(dir)!=NULL,"mkdtemp failed");+ write_fixture(dir);+ Model m; memset(&m,0,sizeof(m)); st_init(&m.S,dir);++ compare_weight(&m,"w8",2,4,8);+ compare_weight(&m,"w4",2,64,4);+ CHECK(k3_mmap_backend_allowed(1,0,0),"CPU-only mapped mode refused");+ CHECK(!k3_mmap_backend_allowed(1,1,0),"mapped Vulkan combination accepted");+ CHECK(!k3_mmap_backend_allowed(1,0,1),"mapped CUDA combination accepted");+ CHECK(k3_mmap_backend_allowed(0,1,1),"default loader changed GPU policy");++ char path[512]; snprintf(path,sizeof(path),"%s/model.safetensors",dir);+ remove(path);+#ifdef _WIN32+ _rmdir(dir);+#else+ rmdir(dir);+#endif+ if(failures){ fprintf(stderr,"k3 mmap: %d failure(s)\n",failures); return 1; }+ puts("k3 mmap: mapped and copied CPU paths are exact"); return 0;+}diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 300e1b50..9e0b04bd 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -314,6 +314,7 @@ Read **only** by `c/kimi_k3.c`. The K3 engine has its own loader, cache and quan| `K3_BITS` | `4` | Expert quantization width. Setting it at all also pins the choice (the engine otherwise infers it from the container). || `K3_MLA_BITS` | `8` | Quantization width for the MLA attention tensors. || `K3_HEAD_BITS` | `8` | Quantization width for the LM head. |+| `K3_MMAP` | `0` (off) | Map fully prepared U8 matrices and F32 sidecars read-only. CPU-only; refuses conversion and enabled GPU backends rather than falling back. || `K3_EXPERT_GB` | `8.0` | RAM budget (GB) for the expert LRU cache; per-layer slots are derived from it. || `K3_LAYERS` | `0` (all) | Load only the first N layers β for smoke tests and trace-only runs. || `K3_MAXT` | `np + ngen` one-shot, `8192` in serve | KV cache capacity in tokens. In serve mode it is also the prompt-rejection bound. |diff --git a/docs/kimi_k3.md b/docs/kimi_k3.mdindex 5653c67d..1405ceac 100644--- a/docs/kimi_k3.md+++ b/docs/kimi_k3.md@@ -111,6 +111,15 @@ on the full model this removes a 10β15 minute quantization pass. Quantizedvalues are bit-identical to the load-time path (verified by hidden-statetrace), so the two formats are numerically interchangeable.+`K3_MMAP=1` is an experimental CPU-only residency option for a fully repacked+container. It maps each prepared U8 matrix and its F32 scale sidecar read-only+instead of copying both into private heap. The mapped pages remain file-backed+and reclaimable; this changes commit/accounting, not the active working set, so+memory pressure can become page faults during inference. The option has no+fallback: tensors that need load-time conversion, non-F32 scale sidecars, and+enabled Vulkan/CUDA backends are refused. A Vulkan build therefore requires+`K3_VK=0 K3_MMAP=1` explicitly.+Sizes: source 1.56 TB β β1.50 TB (`--bits 8`) / β1.48 TB (`--bits 4`). Theexperts (93 % of bytes) are already at 4.25 bits/weight and cannot shrinklosslessly; sub-4-bit expert re-encodes (fmt=5/6) would be double quantization@@ -157,6 +166,7 @@ Judge quantization choices on real-text logits, not synthetic-vector norms.| `K3_BITS` | 4 | load-time bits for KDA/latent/shared/dense mats (4, 8, 32=f32) || `K3_MLA_BITS` | 8 | load-time bits for MLA projections || `K3_HEAD_BITS` | 8 | load-time bits for lm_head |+| `K3_MMAP` | 0 | map fully prepared U8/F32 weights read-only (experimental, CPU-only, no conversion fallback) || `K3_EXPERT_GB` | 8 | routed-expert LRU budget || `K3_VK` | 1 | Vulkan tier when built with `make VK=1 kimi_k3` (0 = pure CPU) || `K3_VK_GB` | driver budget | VRAM cap for the Vulkan tier |