Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(k3): map prepared weights read-only

bherald6e5f36a1657b
diff --git a/c/Makefile b/c/Makefile
index a24cd227..f216e4b6 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -763,6 +763,9 @@ tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unico
tests/test_k3_ram_budget$(EXE): tests/test_k3_ram_budget.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_k3_mmap$(EXE): tests/test_k3_mmap.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h
+ $(CC) $(NOCUDA_CFLAGS) $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)
+
tests/test_tok_kimi_tiny$(EXE): tests/test_tok_kimi_tiny.c tok.h tok_unicode.h tok_unicode_o200k.h json.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index 153e1a99..3bca2ab7 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -39,6 +39,8 @@
* K3_BITS=4|8|32 load-time quant of KDA/latent/shared/dense (default 4)
* K3_MLA_BITS=8|4|32 MLA projections (default 8)
* K3_HEAD_BITS=8|4|32 lm_head (default 8)
+ * K3_MMAP=0|1 map prepared U8 + F32 tensors read-only instead of
+ * copying them to private heap (default 0; CPU-only)
* K3_EXPERT_GB=N routed-expert LRU cache budget (default 8)
* K3_VK=0|1 Vulkan tier (build with `make VK=1 kimi_k3`; default
* 1 when built): shared experts VRAM-resident + a
@@ -110,9 +112,11 @@ typedef struct {
int bos, eos[8], n_eos;
} Cfg;
-/* ---------- RAM-resident weight, quantized at load ---------- */
+/* ---------- RAM-resident or read-only mapped weight ---------- */
typedef struct { int fmt; float *f; int8_t *q8; uint8_t *q4; float *s; int O, I, gs;
- void *vk; /* ColiVkTensor* once uploaded (K3_VK); NULL = CPU only */ } W;
+ void *vk; /* ColiVkTensor* once uploaded (K3_VK); NULL = CPU only */
+ st_mapped_raw data_map, scale_map;
+ int mapped; } W;
typedef struct { /* KDA layer */
W q, k, v, o, g;
@@ -335,6 +339,8 @@ static float w_rowdot(const W *w, int r, const float *x){
#define QCHUNK 1024 /* rows per load-quantize pass */
static int g_bits_env=0; /* K3_BITS explicitly set: enables the
* int8-container -> int4 load downcast */
+static int g_k3_mmap=0; /* prepared U8/F32 tensors stay file-backed */
+static uint64_t g_k3_mmap_bytes=0, g_k3_mmap_views=0;
#ifdef COLI_CUDA
static int g_k3_cuda=0; /* K3_CUDA=1: MXFP4 routed experts on CUDA at decode */
#endif
@@ -342,6 +348,41 @@ static int g_k3_direct=-1; /* K3_DIRECT: O_DIRECT expert reads */
static int g_k3_idot=1; /* K3_IDOT: int8-activation expert matmuls */
static int g_k3_pipe=1; /* K3_PIPE: overlap loads with compute */
static float g_k3_topp=0.f; /* K3_TOPP: routed-expert top-p pruning */
+
+static int k3_mmap_backend_allowed(int mmap_enabled, int vk_enabled, int cuda_enabled){
+ return !mmap_enabled || (!vk_enabled && !cuda_enabled);
+}
+
+static void w_map_prepared(Model *m, W *w, const char *name, const char *scale_name,
+ st_tensor *t, st_tensor *ts){
+ if(ts->dtype!=2){
+ fprintf(stderr,"K3_MMAP=1 requires F32 scale sidecars; %s is %s -- refusing\n",
+ scale_name,st_dtype_name(ts->dtype)); exit(1);
+ }
+ if(st_map_raw(&m->S,name,&w->data_map)!=0){
+ fprintf(stderr,"K3_MMAP=1 could not map %s: %s -- refusing\n",name,strerror(errno)); exit(1);
+ }
+ if(st_map_raw(&m->S,scale_name,&w->scale_map)!=0){
+ int saved=errno; st_unmap_raw(&w->data_map);
+ fprintf(stderr,"K3_MMAP=1 could not map %s: %s -- refusing\n",scale_name,strerror(saved)); exit(1);
+ }
+ if(((uintptr_t)w->scale_map.data%_Alignof(float))!=0){
+ st_unmap_raw(&w->scale_map); st_unmap_raw(&w->data_map);
+ fprintf(stderr,"K3_MMAP=1 scale sidecar %s is not float-aligned -- refusing\n",scale_name); exit(1);
+ }
+ w->mapped=1;
+ w->s=(float*)w->scale_map.data;
+ g_k3_mmap_bytes+=(uint64_t)t->nbytes+(uint64_t)ts->nbytes;
+ g_k3_mmap_views+=2;
+}
+
+static void w_release_host(W *w){
+ if(!w) return;
+ if(w->mapped){ st_unmap_raw(&w->scale_map); st_unmap_raw(&w->data_map); }
+ else { free(w->f); free(w->q8); free(w->q4); free(w->s); }
+ memset(w,0,sizeof(*w));
+}
+
static void w_load(Model *m, W *w, const char *name, int O, int I, int bits){
char nm[512]; snprintf(nm,sizeof(nm),"%s%s",m->pfx,name);
st_tensor *t=st_find(&m->S,nm);
@@ -356,6 +397,10 @@ static void w_load(Model *m, W *w, const char *name, int O, int I, int bits){
if(!ts){ fprintf(stderr,"%s: quantized (U8) but no %s scale sidecar\n",nm,qn); exit(1); }
if(t->nbytes==(int64_t)O*I && ts->numel==O){ /* int8 per-row */
if(g_bits_env && bits==4 && I%64==0){
+ if(g_k3_mmap){
+ fprintf(stderr,"K3_MMAP=1 cannot combine with load-time int8-to-int4 conversion for %s; use a prepared int4 container or unset K3_BITS -- refusing\n",nm);
+ exit(1);
+ }
/* EXPLICIT K3_BITS=4 on an int8 container: downcast to int4-g64
* at load. Halves resident RAM (the 62 GB box cannot hold the
* 93-layer non-expert set at int8 next to a desktop session);
@@ -382,12 +427,21 @@ static void w_load(Model *m, W *w, const char *name, int O, int I, int bits){
free(q8); free(s8);
return;
}
- w->fmt=1; w->q8=malloc((size_t)O*I);
+ w->fmt=1;
+ if(g_k3_mmap){
+ w_map_prepared(m,w,nm,qn,t,ts); w->q8=(int8_t*)w->data_map.data;
+ return;
+ }
+ w->q8=malloc((size_t)O*I);
if(!w->q8){fprintf(stderr,"OOM int8 %s\n",nm);exit(1);}
st_read_raw(&m->S,nm,w->q8,1);
w->s=falloc(O); st_read_f32(&m->S,qn,w->s,0);
} else if(I%64==0 && t->nbytes==(int64_t)O*(I/2) && ts->numel==(int64_t)O*(I/64)){
w->fmt=4; w->gs=64; /* int4-g64 */
+ if(g_k3_mmap){
+ w_map_prepared(m,w,nm,qn,t,ts); w->q4=(uint8_t*)w->data_map.data;
+ return;
+ }
w->q4=malloc((size_t)O*(I/2));
if(!w->q4){fprintf(stderr,"OOM int4 %s\n",nm);exit(1);}
st_read_raw(&m->S,nm,w->q4,1);
@@ -398,6 +452,10 @@ static void w_load(Model *m, W *w, const char *name, int O, int I, int bits){
}
return;
}
+ if(g_k3_mmap){
+ fprintf(stderr,"K3_MMAP=1 requires a fully prepared U8 + F32-sidecar container; %s is %s and would need load-time conversion -- refusing\n",
+ nm,st_dtype_name(t->dtype)); exit(1);
+ }
if(t->numel!=(int64_t)O*I){ fprintf(stderr,"%s: numel %lld != %dx%d\n",nm,(long long)t->numel,O,I); exit(1); }
if(bits>=32){ w->fmt=0; w->f=falloc((int64_t)O*I); st_read_f32(&m->S,nm,w->f,0); return; }
int gs=64;
@@ -549,6 +607,21 @@ static void model_init(Model *m, const char *snap, int n_layers_env){
snprintf(m->pfx,sizeof(m->pfx),"language_model.");
if((c->n_layers+c->res_bs-1)/c->res_bs+1>16){ fprintf(stderr,"attn_res: too many blocks\n"); exit(1); }
g_bits_env = getenv("K3_BITS")!=NULL;
+ g_k3_mmap = getenv("K3_MMAP")?atoi(getenv("K3_MMAP")):0;
+ if(g_k3_mmap){
+ int vk_requested=0, cuda_requested=0;
+#ifdef COLI_VULKAN
+ { const char *ev=getenv("K3_VK"); vk_requested=!ev||atoi(ev); }
+#endif
+#ifdef COLI_CUDA
+ cuda_requested=getenv("K3_CUDA")&&atoi(getenv("K3_CUDA"));
+#endif
+ if(!k3_mmap_backend_allowed(1,vk_requested,cuda_requested)){
+ fprintf(stderr,"K3_MMAP=1 is CPU-only; set K3_VK=0 and K3_CUDA=0 -- refusing before model load\n");
+ exit(1);
+ }
+ fprintf(stderr,"[K3-MMAP] prepared tensor mapping enabled (CPU-only, no conversion fallback)\n");
+ }
#ifdef COLI_CUDA
g_k3_cuda = getenv("K3_CUDA") ? atoi(getenv("K3_CUDA")) : 0;
if(g_k3_cuda){
@@ -657,6 +730,9 @@ static void model_init(Model *m, const char *snap, int n_layers_env){
free(rn); free(rp); }
w_load(m,&m->lm_head,"lm_head.weight",c->vocab,c->hidden,hbits);
} else fprintf(stderr,"[K3] final norm/lm_head not present β€” trace-only mode\n");
+ if(g_k3_mmap)
+ fprintf(stderr,"[K3-MMAP] %llu views, %.1f GB prepared weights/scales file-backed\n",
+ (unsigned long long)g_k3_mmap_views,g_k3_mmap_bytes/1e9);
#ifdef COLI_VULKAN
{ const char *ev=getenv("K3_VK");
if(!ev||atoi(ev)){
diff --git a/c/tests/test_k3_mmap.c b/c/tests/test_k3_mmap.c
new file mode 100644
index 00000000..a4c92da3
--- /dev/null
+++ b/c/tests/test_k3_mmap.c
@@ -0,0 +1,79 @@
+/* K3_MMAP: prepared U8 weights and F32 scales must produce exactly the same
+ * CPU math as the existing heap-copy loader while remaining file-backed. */
+#define main kimi_k3_main_unused
+#include "../kimi_k3.c"
+#undef main
+
+#include <stdio.h>
+
+static int failures;
+#define CHECK(cond, ...) do { if(!(cond)){ \
+ fprintf(stderr,"FAIL %s:%d: ",__FILE__,__LINE__); \
+ fprintf(stderr,__VA_ARGS__); fputc('\n',stderr); failures++; } } while(0)
+
+static void write_fixture(const char *dir){
+ char path[512]; snprintf(path,sizeof(path),"%s/model.safetensors",dir);
+ const char *hdr=
+ "{\"w8\":{\"dtype\":\"U8\",\"shape\":[2,4],\"data_offsets\":[0,8]},"
+ "\"w8.qs\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[8,16]},"
+ "\"w4\":{\"dtype\":\"U8\",\"shape\":[2,32],\"data_offsets\":[16,80]},"
+ "\"w4.qs\":{\"dtype\":\"F32\",\"shape\":[2,1],\"data_offsets\":[80,88]}}";
+ const int8_t q8[8]={-7,3,9,12,5,-4,2,11};
+ const float s8[2]={0.25f,1.5f};
+ uint8_t q4[64]; float s4[2]={0.125f,0.75f};
+ for(int i=0;i<64;i++) q4[i]=(uint8_t)((i*13+7)&255);
+ size_t raw_hlen=strlen(hdr);
+ size_t pad=(8-((8+raw_hlen)%8))%8;
+ uint64_t hlen=(uint64_t)(raw_hlen+pad);
+ FILE *f=fopen(path,"wb");
+ fwrite(&hlen,8,1,f); fwrite(hdr,1,raw_hlen,f);
+ for(size_t i=0;i<pad;i++) fputc(' ',f);
+ fwrite(q8,1,sizeof(q8),f); fwrite(s8,1,sizeof(s8),f);
+ fwrite(q4,1,sizeof(q4),f); fwrite(s4,1,sizeof(s4),f); fclose(f);
+}
+
+static void compare_weight(Model *m, const char *name, int O, int I, int bits){
+ W copied={0}, mapped={0}; float x[64], yc[2]={0}, ym[2]={0};
+ float ac[64]={0}, am[64]={0};
+ for(int i=0;i<I;i++) x[i]=(float)(i%9-4)*0.125f;
+
+ g_k3_mmap=0; g_bits_env=0; w_load(m,&copied,name,O,I,bits);
+ g_k3_mmap=1; w_load(m,&mapped,name,O,I,bits);
+ CHECK(mapped.mapped,"%s did not use mapped storage",name);
+ CHECK(!copied.mapped,"%s changed the default loader",name);
+ CHECK(mapped.s==(float*)mapped.scale_map.data,"%s scales are not mapped",name);
+ if(mapped.fmt==1) CHECK(mapped.q8==(int8_t*)mapped.data_map.data,"%s int8 data are not mapped",name);
+ if(mapped.fmt==4) CHECK(mapped.q4==(uint8_t*)mapped.data_map.data,"%s int4 data are not mapped",name);
+
+ w_matmul(yc,x,&copied,1); w_matmul(ym,x,&mapped,1);
+ CHECK(!memcmp(yc,ym,sizeof(yc)),"%s matmul output differs",name);
+ w_addrow(&copied,1,0.75f,ac); w_addrow(&mapped,1,0.75f,am);
+ CHECK(!memcmp(ac,am,(size_t)I*sizeof(float)),"%s row expansion differs",name);
+ float dc=w_rowdot(&copied,1,x), dm=w_rowdot(&mapped,1,x);
+ CHECK(!memcmp(&dc,&dm,sizeof(dc)),"%s row dot differs",name);
+
+ w_release_host(&mapped); w_release_host(&copied);
+}
+
+int main(void){
+ char dir[]="test_k3_mmap_XXXXXX"; CHECK(mkdtemp(dir)!=NULL,"mkdtemp failed");
+ write_fixture(dir);
+ Model m; memset(&m,0,sizeof(m)); st_init(&m.S,dir);
+
+ compare_weight(&m,"w8",2,4,8);
+ compare_weight(&m,"w4",2,64,4);
+ CHECK(k3_mmap_backend_allowed(1,0,0),"CPU-only mapped mode refused");
+ CHECK(!k3_mmap_backend_allowed(1,1,0),"mapped Vulkan combination accepted");
+ CHECK(!k3_mmap_backend_allowed(1,0,1),"mapped CUDA combination accepted");
+ CHECK(k3_mmap_backend_allowed(0,1,1),"default loader changed GPU policy");
+
+ char path[512]; snprintf(path,sizeof(path),"%s/model.safetensors",dir);
+ remove(path);
+#ifdef _WIN32
+ _rmdir(dir);
+#else
+ rmdir(dir);
+#endif
+ if(failures){ fprintf(stderr,"k3 mmap: %d failure(s)\n",failures); return 1; }
+ puts("k3 mmap: mapped and copied CPU paths are exact"); return 0;
+}
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 300e1b50..9e0b04bd 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -314,6 +314,7 @@ Read **only** by `c/kimi_k3.c`. The K3 engine has its own loader, cache and quan
| `K3_BITS` | `4` | Expert quantization width. Setting it at all also pins the choice (the engine otherwise infers it from the container). |
| `K3_MLA_BITS` | `8` | Quantization width for the MLA attention tensors. |
| `K3_HEAD_BITS` | `8` | Quantization width for the LM head. |
+| `K3_MMAP` | `0` (off) | Map fully prepared U8 matrices and F32 sidecars read-only. CPU-only; refuses conversion and enabled GPU backends rather than falling back. |
| `K3_EXPERT_GB` | `8.0` | RAM budget (GB) for the expert LRU cache; per-layer slots are derived from it. |
| `K3_LAYERS` | `0` (all) | Load only the first N layers β€” for smoke tests and trace-only runs. |
| `K3_MAXT` | `np + ngen` one-shot, `8192` in serve | KV cache capacity in tokens. In serve mode it is also the prompt-rejection bound. |
diff --git a/docs/kimi_k3.md b/docs/kimi_k3.md
index 5653c67d..1405ceac 100644
--- a/docs/kimi_k3.md
+++ b/docs/kimi_k3.md
@@ -111,6 +111,15 @@ on the full model this removes a 10–15 minute quantization pass. Quantized
values are bit-identical to the load-time path (verified by hidden-state
trace), so the two formats are numerically interchangeable.
+`K3_MMAP=1` is an experimental CPU-only residency option for a fully repacked
+container. It maps each prepared U8 matrix and its F32 scale sidecar read-only
+instead of copying both into private heap. The mapped pages remain file-backed
+and reclaimable; this changes commit/accounting, not the active working set, so
+memory pressure can become page faults during inference. The option has no
+fallback: tensors that need load-time conversion, non-F32 scale sidecars, and
+enabled Vulkan/CUDA backends are refused. A Vulkan build therefore requires
+`K3_VK=0 K3_MMAP=1` explicitly.
+
Sizes: source 1.56 TB β†’ β‰ˆ1.50 TB (`--bits 8`) / β‰ˆ1.48 TB (`--bits 4`). The
experts (93 % of bytes) are already at 4.25 bits/weight and cannot shrink
losslessly; sub-4-bit expert re-encodes (fmt=5/6) would be double quantization
@@ -157,6 +166,7 @@ Judge quantization choices on real-text logits, not synthetic-vector norms.
| `K3_BITS` | 4 | load-time bits for KDA/latent/shared/dense mats (4, 8, 32=f32) |
| `K3_MLA_BITS` | 8 | load-time bits for MLA projections |
| `K3_HEAD_BITS` | 8 | load-time bits for lm_head |
+| `K3_MMAP` | 0 | map fully prepared U8/F32 weights read-only (experimental, CPU-only, no conversion fallback) |
| `K3_EXPERT_GB` | 8 | routed-expert LRU budget |
| `K3_VK` | 1 | Vulkan tier when built with `make VK=1 kimi_k3` (0 = pure CPU) |
| `K3_VK_GB` | driver budget | VRAM cap for the Vulkan tier |