Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
/* Kimi K3 inference engine in pure C — sibling of colibri.c (GLM-5.2) / olmoe.c /
* inkling.c, sharing st.h / json.h / tok.h / quant.h.
*
* Architecture (2.8T total / 104B active, 93 layers, hidden 7168):
* - Hybrid attention: 69 KDA (Kimi Delta Attention, linear/recurrent) +
* 24 gated MLA layers (every 4th, plus the final layer). MLA is NoPE —
* no positional encoding anywhere in the model; position lives in KDA's
* decay/conv. Both attention types carry a full-rank sigmoid output gate.
* - AttnRes (Attention Residuals) REPLACE the plain residual stream: a
* running prefix_sum plus block snapshots (layers 0,12,24,...,84), mixed
* by softmax twice per layer (before attention, before MLP) and once at
* the end. Weights: per-layer {self_attention,mlp}_res_{norm,proj}.
* - Stable LatentMoE: router (sigmoid + e_score_correction_bias, top-16 of
* 896, renormalized raw scores), shared latent down/up projections
* (7168<->3584), per-expert GLU in the 3584 latent with moe_inter 3072,
* RMSNorm on the aggregate, plus 2 fused shared experts (inter 6144) at
* full width. Activation is SiTU-GLU:
* b1*tanh(g/b1)*sigmoid(g) * b2*tanh(u/b2), b1=4, b2=25.
* - Routed experts are NATIVE MXFP4 (QAT; e2m1 nibbles + ue8m0 scale per
* 32, compressed-tensors "mxfp4-pack-quantized") and are streamed
* straight from the original HF shards — never re-encoded, never
* converted. Everything else is BF16 in the checkpoint and quantized at
* LOAD TIME into RAM (int8 per-row / int4-g64 / f32, see K3_*BITS).
*
* KDA per-head recurrence (head dim 128, 96 heads; fla fused_recurrent_kda):
* q,k,v = SiLU(ShortConv4(W{q,k,v} x)); q,k L2-normalized (eps 1e-6
* inside the sqrt), q *= 128^-0.5
* z = W_fb(W_fa x) + dt_bias (per channel, dt_bias[12288])
* gk = gmin * sigmoid(exp(A_log[h]) * z), gmin=-5; alpha = exp(gk)
* S = (I - beta k k^T) Diag(alpha) S + beta k v^T, beta = sigmoid(W_b x)
* o = S^T q; out = W_o [ sigmoid(W_g x) * RMSNorm_head(o) ]
* (checkpoint A_log is [128] = per-head [96] zero-padded, first 96 used)
*
* Model dir = the HF snapshot (config.json + model-*-of-000096.safetensors).
* tokenizer.json is synthesized once by tools/k3_tokenizer.py (the HF repo
* ships only tiktoken.model); without it the engine still runs on raw ids.
*
* ENV:
* K3_BITS=4|8|32 load-time quant of KDA/latent/shared/dense (default 4)
* K3_MLA_BITS=8|4|32 MLA projections (default 8)
* K3_HEAD_BITS=8|4|32 lm_head (default 8)
* K3_MMAP=0|1 map prepared U8 + F32 tensors read-only instead of
* copying them to private heap (default 0; CPU-only)
* K3_EXPERT_GB=N routed-expert LRU cache budget (default 8)
* COLI_VULKAN=1 (build with `make VK=1 kimi_k3`) the routed experts
* on the shared Vulkan expert tier (vk_tier.c, the
* COLI_VK_TIER* variables), the shared experts where
* COLI_VK_DENSE puts the dense matrices. CPU fallback
* everywhere; the tier's experts match K3_IDOT=0.
* COLI_VK_CHAIN=0|1|2 with COLI_VULKAN=1: every layer's dense part on the
* device, one frame per layer up to the router
* (kimi_k3_chain.h; 2: prompts only). Default: on a
* discrete GPU, off on an integrated one (not measured).
* K3_VK=0|1 the old switch, an alias: 1 = COLI_VULKAN=1, 0 = the
* device stays closed. Unset: COLI_VULKAN decides.
* K3_VK_GB=N alias of COLI_VK_TIER_GB (when that is unset)
* K3_VK_UP=N alias of COLI_VK_TIER_RATE (when that is unset);
* `auto` and K3_VK_FILL_FRAC do nothing any more
* K3_METAL=0|1 Metal tier (build with `make METAL=1 kimi_k3`; Phase 4:
* scaffolding only — dispatch hooks present, forward NOT
* IMPLEMENTED; all ops fall through to CPU).
* K3_DIRECT=0|1 O_DIRECT expert reads (default 1; buffered fallback)
* K3_IDOT=0|1 int8-activation expert matmuls (default 1; 0 = float)
* K3_PIPE=0|1 overlap expert loads with compute (default 1)
* K3_LOAD_THREADS=N loader threads for K3_PIPE (default 4)
* K3_TOPP=F keep routed experts to cumulative weight F (0 = off)
* K3_CHUNK=N prefill chunk size (default 32; 1 = token-at-a-time)
* K3_THINK=0|1 chat mode: open the think channel (default 1)
* K3_LAYERS=N truncate to first N layers (validation; skips head)
* K3_TRACE=path dump f32 hidden state after every layer (validation)
* K3_LOGITS=path dump f32 logits per PREFILL position (teacher-forced
* bit-width comparisons; use with --ngen 0)
* K3_MAXT=N KV/context capacity (default prompt+ngen)
* COLI_K3_CKPT=N OPT-IN recurrent-state checkpoints for serve: keep N
* photos of the KDA state at turn boundaries so a
* divergent prompt (agentic edit) resumes from the
* deepest valid photo instead of re-prefilling from
* token 0. Unset/0 (default) = feature off, engine
* byte-identical to before. ~434 MB per photo on the
* full model, which is why it is opt-in.
* COLI_K3_CKPT_DIR=dir park the photos in files under dir instead of RAM
* (low-RAM machines: ~0.4s of sequential I/O per
* restore against minutes of avoided re-prefill).
* Alone it implies COLI_K3_CKPT=2; an explicit
* COLI_K3_CKPT (0 included) always wins. Photos never
* outlive the process: after a restart the MLA rows
* are gone, so no restore could be valid.
* COLI_TEMP=F 0 = greedy (default), else softmax temperature
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <time.h>
#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
#include <sys/resource.h>
#include <sys/select.h>
#include <unistd.h>
#endif
#include <pthread.h>
#include <stdatomic.h>
#ifdef _OPENMP
#include <omp.h>
#endif
#ifdef __APPLE__
#include <sys/sysctl.h>
#include <mach/mach.h>
#endif
#include "cli_args.h"
#include "st.h"
#include "tok.h"
#include "quant.h"
#ifdef COLI_CUDA
#include "backend_cuda.h"
#endif
#include "omp_tune.h"
#include "route_trace.h"
#include "vk_tier.h" /* the shared Vulkan routed-expert tier (COLI_VULKAN=1; stubs without VK=1) */
#include "kv_prefix.h"
#include "pin_pool.h" /* coli_pin_slots_wanted: quanti scatti tenere */
#include "hybrid_split.h" /* KV prefix reuse (shared) */
#include "serve_codec.h"
#include "serve_budget.h"
#ifdef COLI_SEGMENT_ADAPTER
#include "segment_runtime.h"
#include "segment_adapters.h"
#include "segment_adapter_internal.h"
#endif
#ifdef COLI_EDGE_ADAPTER
#include "edge_runtime.h"
#include "edge_adapters.h"
#include "edge_tok_internal.h"
#endif
/* ---------- config ---------- */
typedef struct {
int hidden, n_layers, vocab, first_dense, dense_inter;
/* MLA */
int n_heads, q_lora, kv_lora, qk_nope, qk_rope, qk_head, v_head;
float attn_scale, theta;
/* KDA */
int kda_heads, kda_hd, kda_proj, conv_k;
float gate_lb;
/* MoE */
int n_experts, topk, moe_inter, latent, n_shared;
float situ_b1, situ_b2;
/* AttnRes */
int res_bs;
float eps;
int8_t is_kda[128];
int8_t idx_type[128]; /* DSA: 1=full (own Ic + top-K), 0=shared (reuse) */
int index_hd, index_nh, index_topk; /* DSA indexer dimensions */
int bos, eos[8], n_eos;
} Cfg;
/* ---------- RAM-resident or read-only mapped weight ---------- */
typedef struct { int fmt; float *f; int8_t *q8; uint8_t *q4; float *s; int O, I, gs;
void *vk; /* ColiVkTensor* once uploaded (K3_VK); NULL = CPU only */
void *metal; /* ColiMetalTensor* once registered (K3_METAL); NULL = CPU only */
st_mapped_raw data_map, scale_map;
int mapped;
/* COLI_VULKAN: how to read it back from disk (k3_w_reload), and 1 while the
* device holds it alone (COLI_VK_DENSE_HOST, the host copy dropped) */
char *vk_name; int vk_bits, vk_gone; } W;
typedef struct { /* KDA layer */
W q, k, v, o, g;
float *conv_q, *conv_k, *conv_v; /* [proj*4] depthwise taps, oldest first */
float *fa, *fb; /* decay low-rank, f32 [hd,hidden] [proj,hd] */
float *bp; /* beta proj f32 [heads,hidden] */
float *dt, *A, *onw; /* dt_bias[proj], exp(A_log)[heads], o_norm[hd] */
void *metal_fa, *metal_fb, *metal_bp; /* Metal tensor wrappers for f32 weights */
void *vk_fa, *vk_fb, *vk_bp; /* COLI_VK_DENSE_HOST: the chain's device copies, for the per-matrix path */
} Kda;
typedef struct { /* gated MLA layer */
W qa, qb, kva, kvb, o, g;
float *qa_ln, *kva_ln;
/* DSA indexer (full layers only, optional) */
W wk, wq, wp; /* [index_hd x {hidden, q_lora, index_hd}] */
float *knw, *knb; /* key layernorm [index_hd] */
float *Ic; /* indexer cache [max_t * index_hd] */
} Mla;
typedef struct { /* LatentMoE */
float *router, *rbias, *lat_norm; /* [E,hidden] f32, [E], [latent] */
void *vk_router; /* COLI_VK_DENSE_HOST: the chain's device copy, for the per-matrix path */
W lat_down, lat_up, sh_gate, sh_up, sh_down;
} Moe;
typedef struct {
int kda, sparse;
Kda a; Mla m; Moe moe;
W d_gate, d_up, d_down; /* dense layer only */
float *in_ln, *post_ln;
float *attn_sw, *mlp_sw; /* AttnRes score weights: norm.w * proj.w */
} Layer;
/* ---------- routed-expert streaming (native MXFP4 from the HF shards) ---- */
typedef struct { int fd[6]; int64_t off[6]; int contig; } ERef; /* w1p w1s w2p w2s w3p w3s */
#ifndef KIMI_K3_NO_MAIN
static char g_k3_usage[2100]; /* <snap>/.coli_usage, or COLI_USAGE */
#endif
typedef struct { int eid; uint8_t *buf, *base; uint64_t used; int pinned; } Slot;
/* pinned: seeded from .coli_usage at startup and never evicted. The LRU adapts to
* THIS session; the pin knows the history of every session before it. Capped at
* half the layer budget so the adaptive half always survives — an all-pinned cache
* that guessed wrong is slower than no pin at all (measured on GLM: 0.17 vs 0.25
* tok/s when the pins came from a single prompt). */
/* base = 4K-aligned allocation (O_DIRECT target);
* buf = expert data view inside it (= base + off%4K) */
typedef struct {
Slot *s;
int *slot_by_expert; /* expert id -> local slot, -1 if absent */
int n, cap;
} LCache;
/* One conversation's state for a multiplexed serve (KV_SLOTS>1, serve_mux below):
* every KDA layer's recurrent state and its three convolution windows, every MLA
* layer's latent caches and its index keys, their capacity and the record of the
* tokens they hold. The Model holds the conversation a prefill runs on (k3_seq_swap
* trades it for a parked one); a multiplexed decode step parks them all and reads
* each row's from its K3Row. */
typedef struct {
float **kstate, **cwq, **cwk, **cwv, **Lc, **Rc, **Ic;
int max_t;
kv_prefix kvp;
} K3Seq;
typedef struct { K3Seq *seq; int pos; } K3Row;
static int g_k3_mux_slots=1; /* KV_SLOTS: the conversations a serve decodes at once */
typedef struct {
Cfg c;
shards S;
char pfx[40]; /* "language_model." or "" */
Layer *L;
float *final_norm, *out_sw;
W lm_head;
int has_head;
Slot ws[64]; /* working set: parallel loads land here,
* then swap into the layer LRU */
/* KDA state */
float **kstate; /* [layer] -> [heads*hd*hd], S[k][v] */
float **cwq, **cwk, **cwv; /* conv windows [proj*conv_k], oldest first */
/* MLA cache */
float **Lc, **Rc; int max_t;
/* KV prefix reuse: what the current state was built from (kv_prefix.h).
* K3 has no single KV to inspect — 69 KDA layers carry a RECURRENT state
* and only the 24 MLA layers keep Lc/Rc — so an explicit record of the
* tokens fed is the only description of it that cannot drift. */
kv_prefix kvp;
/* DSA selection: per-slot k_idx arrays (shared across layers) */
int *dsa_nsel, *dsa_sel; int64_t dsa_scap; /* k_idx selection: [S] counts + [S*topk] indices */
/* experts */
ERef *eref; /* [n_layers][n_experts] (dense rows zeroed) */
LCache *ecache;
uint8_t **ehit; /* experts routed this turn, for HITS (dashboard Brain) */
int64_t e_w1p, e_w1s, e_w2p, e_w2s, e_slot;
uint64_t clock, hits, miss, ebytes;
double t_attn, t_moe, t_eload, t_head;
FILE *trace;
/* A multiplexed decode step (k3_step_rows): row t is the token at
* mux_rows[t].pos of mux_rows[t].seq; NULL in every other forward. */
const K3Row *mux_rows;
} Model;
static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+t.tv_nsec*1e-9; }
/* How many expert slots per layer fit a budget. PURE -- no globals, no model,
* no I/O -- so the arithmetic that #855 got wrong can be tested against the
* reported numbers without a 600 GB checkpoint. Returns the cap; writes the
* bytes left for experts through `for_experts_out` when non-NULL.
*
* reserve = page cache + activations + KV + optional recurrent checkpoints,
* all in GB. `cap_requested` is what K3_EXPERT_GB asked for: this only ever
* LOWERS it, never raises it, so an explicit small cache stays small. */
static int k3_cap_for_ram(double budget_gb, double resident_gb, double reserve_gb,
double slot_gb, int nmoe, int cap_requested,
int n_experts, double *for_experts_out){
double for_experts = budget_gb - resident_gb - reserve_gb;
if(for_experts_out) *for_experts_out = for_experts;
if(nmoe < 1) nmoe = 1;
if(!(slot_gb > 0.0)) return cap_requested;
int fits = for_experts > 0.0 ? (int)(for_experts/(slot_gb*(double)nmoe)) : 0;
if(fits > n_experts) fits = n_experts;
return fits < cap_requested ? fits : cap_requested;
}
/* Memory the OS says is still reclaimable without swapping, in GB; 0 if unknown.
* The same quantity colibri.c's cap_for_ram() budgets against -- Linux
* MemAvailable, Windows ullAvailPhys, macOS free+inactive+purgeable. #855. */
static double k3_mem_avail(void){
#ifdef _WIN32
double total=0, avail=0; compat_meminfo(&total,&avail); return avail;
#elif defined(__APPLE__)
int64_t pgsz=0; size_t sl=sizeof(pgsz);
if(sysctlbyname("hw.pagesize",&pgsz,&sl,NULL,0)||pgsz<=0) pgsz=16384;
vm_statistics64_data_t vs; mach_msg_type_number_t nc=HOST_VM_INFO64_COUNT;
if(host_statistics64(mach_host_self(),HOST_VM_INFO64,(host_info64_t)&vs,&nc)!=KERN_SUCCESS)
return 0;
return (double)(vs.free_count+vs.inactive_count+vs.purgeable_count)*(double)pgsz/1e9;
#else
FILE *f=fopen("/proc/meminfo","r"); if(!f) return 0;
char ln[256]; double kb=0;
while(fgets(ln,sizeof(ln),f)) if(sscanf(ln,"MemAvailable: %lf",&kb)==1) break;
fclose(f); return kb/1e6;
#endif
}
static double rss_gb(void){ struct rusage r; getrusage(RUSAGE_SELF,&r);
#if defined(__APPLE__)
return r.ru_maxrss/(1024.0*1024.0*1024.0);
#else
return r.ru_maxrss/(1024.0*1024.0);
#endif
}
static float *falloc(int64_t n){
if(n<0 || (uint64_t)n > SIZE_MAX/sizeof(float)){ fprintf(stderr,"alloc size overflow: %lld floats\n",(long long)n); exit(1); }
float *p=malloc((size_t)n*sizeof(float)); if(!p){fprintf(stderr,"OOM %lld floats\n",(long long)n);exit(1);} return p; }
static float *fcalloc(int64_t n){
if(n<0 || (uint64_t)n > SIZE_MAX/sizeof(float)){ fprintf(stderr,"alloc size overflow: %lld floats\n",(long long)n); exit(1); }
float *p=calloc((size_t)n,sizeof(float)); if(!p){fprintf(stderr,"OOM %lld floats\n",(long long)n);exit(1);} return p; }
/* Aligned + zeroed alloc. Metal's wrap() only takes the zero-copy (newBufferWithBytesNoCopy)
* path when the pointer AND size are 16384-aligned; otherwise it makes a private GPU copy
* that is never synced back. Buffers the GPU WRITES and must persist across tokens (KDA
* state, conv windows) therefore have to be 16 KB-aligned so GPU updates land in host memory.
* Freed with plain free(); posix_memalign memory is free()-compatible on macOS. */
static float *afcalloc(int64_t n){
#ifdef COLI_METAL
size_t sz=(size_t)n*sizeof(float); void *p=NULL;
if(posix_memalign(&p,16384,sz)){ fprintf(stderr,"OOM %lld floats\n",(long long)n); exit(1); }
memset(p,0,sz); return (float*)p;
#else
return fcalloc(n);
#endif
}
static inline float sigmoidf_(float x){ return 1.f/(1.f+expf(-x)); }
static inline float siluf_(float x){ return x/(1.f+expf(-x)); }
static void softmax_(float *x, int n){ float m=x[0]; for(int i=1;i<n;i++) if(x[i]>m)m=x[i];
float s=0; for(int i=0;i<n;i++){ x[i]=expf(x[i]-m); s+=x[i]; } for(int i=0;i<n;i++) x[i]/=s; }
static void rmsnorm_(float *out, const float *x, const float *w, int D, float eps){
double ms=0; for(int i=0;i<D;i++) ms+=(double)x[i]*x[i];
float r=1.f/sqrtf((float)(ms/D)+eps);
for(int i=0;i<D;i++) out[i]=x[i]*r*w[i];
}
/* ---------- DSA (Dictionary Sparse Attention) CPU helpers ---------- */
/* L'indexer DSA e' COSTRUITO ma non COLLEGATO (#1333): l'unica funzione che
* leggerebbe la cache Ic per scegliere le posizioni top-K, dsa_score_single qui
* sotto, non ha nessun chiamante -- una grep su tutto il repo trova solo la sua
* definizione.
*
* Finche' resta cosi' l'attenzione e' densa e il sottosistema e' lavoro pagato e
* mai letto: CINQUE tensori dal disco per ogni layer DSA (w_k, w_q, w_p, kn_w,
* kn_b -- e w_q, w_p, kn_b non sono toccati nemmeno dal codice morto), la cache
* Ic da [max_t * index_hd] float per layer, e per ogni token un matmul piu'
* rmsnorm piu' rope.
*
* Spento di default, non cancellato: quel che manca e' collegare
* dsa_score_single, e chi lo fara' ha bisogno di questi pesi. KIMI_DSA_INDEXER=1
* li ricarica. Avvertenza per chi ci mettera' mano: collegarlo CAMBIA le uscite
* -- l'attenzione diventa sparsa dove oggi e' densa -- quindi va fatto contro
* l'oracolo token-exact, non a occhio. */
static int k3_dsa_indexer_on(void){
static int cached = -1;
if(cached < 0){ const char *e = getenv("KIMI_DSA_INDEXER"); cached = e && *e=='1'; }
return cached;
}
static void dsa_rope(float *v, int pos, int qk_rope, float base_theta){
int half = qk_rope/2;
if(qk_rope > 256){ fprintf(stderr,"qk_rope=%d exceeds rope buffer (256)\n",qk_rope); exit(1); }
float in[256]; memcpy(in,v,qk_rope*sizeof(float));
for(int j=0;j<half;j++){
float inv=powf(base_theta,-2.0f*j/qk_rope), ang=pos*inv;
float cs=cosf(ang), sn=sinf(ang);
float a=in[2*j], b=in[2*j+1];
v[j] = a*cs - b*sn;
v[half+j] = b*cs + a*sn;
}
}
typedef struct { float sc; int idx; } DsaEntry;
static int dsa_entry_cmp_desc(const void *a,const void *b){
float d=((const DsaEntry*)b)->sc-((const DsaEntry*)a)->sc;
return d>0?1:d<0?-1:0;
}
/* dsa_score: multi-head cosine distance + ReLU + weight + top-K select.
* qi: [nh * index_hd] — per-head indexer query (post-RoPE, post-projection)
* Ic: [nk * index_hd] — indexer cache rows
* w32: [nh] — weight scalars from P projection
* nk: number of cache rows in context
* sel: output buffer of maxsel ints (sorted desc by score)
* Returns actual count of selected indices. Caller supplies token-local DsaEntry array. */
static int dsa_score_single(int *sel, int maxsel, const float *qi, const float *Ic, int index_hd,
const float *w32, int nh, int nk, DsaEntry *entries){
if(nk <= 0 || maxsel <= 0) return 0;
for(int t=0;t<nk;t++){
float s = 0;
for(int h=0;h<nh;h++){
float d=0;
for(int i=0;i<index_hd;i++) d += qi[(int64_t)h*index_hd+i] * Ic[(int64_t)t*index_hd+i];
s += w32[h]*(d>0?d:0); /* ReLU per-head */
}
entries[t]=(DsaEntry){s/sqrtf((float)nh), t};
}
int K = nk < maxsel ? nk : maxsel;
qsort(entries, nk, sizeof(DsaEntry), dsa_entry_cmp_desc);
for(int k=0;k<K;k++) sel[k]=entries[k].idx;
return K;
}
/* ---------- W: load-time quantization + matvec ---------- */
/* ---------- Vulkan (COLI_VULKAN=1, build with `make VK=1 kimi_k3`) ----------
* Two things on the device, each with the CPU path as its fallback:
* - the routed experts, on the shared routed-expert tier (vk_tier.c): the
* checkpoint's MXFP4 with its ue8m0 group scales, SiTU-GLU on the device, in
* the latent space (the tier's hidden is the latent, its inter moe_inter). It
* fills from the expert history and from the experts the CPU computes, and
* evicts the coldest as the routing moves (moe_forward, k3_vk_tier_start);
* - the shared experts' three matrices, where coli_vk_dense() puts the dense
* matrices (on a discrete GPU the device; on one that shares the CPU's RAM,
* the CPU while the tier is on), run by w_matmul one row at a time (S == 1).
* K3_VK, K3_VK_GB and K3_VK_UP were the switches of this engine's own tier
* before the shared one; k3_vk_aliases() reads them into the shared names. */
#ifdef COLI_VULKAN
#include "backend_vulkan.h"
#include "vk_chain.h" /* vkc_fit: how many layers the dense chain places (a partial chain) */
static int g_k3_vk=0; /* the device is open */
/* The partial chain (kimi_k3_chain.h, k3c_fit_now): the first g_k3_fit.n layers on the
* device, decided once before anything goes up and before the expert cache is sized.
* g_k3_partial: something the full chain would place stays on the CPU (layers, the head),
* so nothing more goes to the device for the per-matrix path. g_k3_dho_layers and
* g_k3_dho_head: what COLI_VK_DENSE_HOST takes off the host (the chain's layers, and the
* head only with every layer and the tail on the device). */
static VkcFit g_k3_fit;
static int g_k3_fit_done, g_k3_partial;
static int g_k3_dho_layers=1<<30, g_k3_dho_head=1;
/* COLI_VK_DENSE_HOST: the dense matrices on the device only (k3_dho_*): every W the
* chain does not run goes to the device through w_matmul, prompts included */
static int g_k3_dho=0;
static void k3_w_reload(W *w); /* below, beside w_load */
static int k3_in_par(void){
#ifdef _OPENMP
return omp_in_parallel();
#else
return 0;
#endif
}
/* Did the run ask for the device? K3_VK, when set, decides (0: never, any other
* number: yes, as COLI_VULKAN=1); unset, COLI_VULKAN does, as for every engine. */
static int k3_vk_requested(void){
const char *k=getenv("K3_VK");
if(k&&*k) return atoi(k)!=0;
const char *v=getenv("COLI_VULKAN");
return v&&*v&&atoi(v)!=0;
}
/* The old switches, read as the shared tier's (an explicit shared one wins):
* K3_VK=1 -> COLI_VULKAN=1 (K3_VK=0 keeps the device closed)
* K3_VK_GB=N -> COLI_VK_TIER_GB=N (the routed experts' budget)
* K3_VK_UP=N -> COLI_VK_TIER_RATE=N (promotions per token); `auto` and
* K3_VK_FILL_FRAC have nothing left to bound: the shared
* tier uploads on a thread of its own and never holds a step */
static void k3_vk_aliases(void){
const char *k=getenv("K3_VK"), *v=getenv("COLI_VULKAN");
if(k&&*k&&atoi(k)&&!(v&&*v&&atoi(v))){
setenv("COLI_VULKAN","1",1);
fprintf(stderr,"[VK] kimi_k3: K3_VK=%s read as COLI_VULKAN=1\n",k);
}
const char *gb=getenv("K3_VK_GB"), *tg=getenv("COLI_VK_TIER_GB");
if(gb&&*gb&&atof(gb)>0&&!(tg&&*tg)){
setenv("COLI_VK_TIER_GB",gb,1);
fprintf(stderr,"[VK] kimi_k3: K3_VK_GB=%s read as COLI_VK_TIER_GB=%s\n",gb,gb);
}
const char *up=getenv("K3_VK_UP"), *tr=getenv("COLI_VK_TIER_RATE");
if(up&&*up&&!(tr&&*tr)){
if(!strcmp(up,"auto"))
fprintf(stderr,"[VK] kimi_k3: K3_VK_UP=auto has no effect on the shared tier (its uploads never hold a step)\n");
else {
setenv("COLI_VK_TIER_RATE",up,1);
fprintf(stderr,"[VK] kimi_k3: K3_VK_UP=%s read as COLI_VK_TIER_RATE=%s\n",up,up);
}
}
}
/* the backend format of a resident W: int8 rows, int4 groups, f32 */
static int k3_vk_fmt(const W *w){ return w->fmt==1?1 : w->fmt==4?4 : w->fmt==0?10 : -1; }
static int w_vk_upload(W *w){
if(w->vk) return 1;
int fmt=k3_vk_fmt(w);
if(fmt<0||w->mapped) return 0;
const void *src=fmt==1?(const void*)w->q8 : fmt==4?(const void*)w->q4 : (const void*)w->f;
return coli_vk_tensor_ensure((ColiVkTensor**)&w->vk,src,fmt==10?NULL:w->s,fmt,w->I,w->O,fmt==4?w->gs:0);
}
#endif
/* ---------- Metal tier (K3_METAL, build with `make METAL=1 kimi_k3`) ----------
* Phase 4: scaffolding — init, dispatch hooks, feature flags.
* Not implemented; w_matmul intentionally falls through to CPU. */
static int g_k3_metal=0; /* backend live (K3_METAL env) */
/* Phase 9: layer-level validation — capture intermediates for one layer */
static int g_k3_val_layer = -1; /* K3_VALIDATE_LAYER=N (0-indexed, -1=off) */
static int g_k3_val_token = 0; /* K3_VALIDATE_TOKEN (always use 0) */
static FILE *g_k3_val_fp = NULL; /* output file for intermediate dumps */
/* Phase 10: full model validation — per-step logits (stays open across prefill+decode) */
static FILE *g_k3_val_lfp = NULL; /* K3_VAL_LOGITS: per-step logit dump */
static void val_dump(const char *name, const float *v, int n); /* forward-declare */
static FILE *g_k3_dfp = NULL; /* K3_DEBUG_OUT: Metal/CPU side-by-side per-dim comparison */
#ifdef COLI_METAL
#include "backend_metal.h"
/* Metal forward entry point — Phase 4: NOT IMPLEMENTED */
static int kimima_forward_metal(Model *m, Layer *l, int li, const float *x, int C, float *out){
(void)m; (void)l; (void)li; (void)x; (void)C; (void)out;
return 0; /* 0 = NOT IMPLEMENTED — fall through to CPU */
}
#endif
static void w_matmul(float *y, const float *x, const W *w, int S){
#ifdef COLI_METAL
if(g_k3_metal && coli_metal_available()){
if(w->fmt==0){
/* fmt=0 is raw f32, no scales needed — Metal shader handles it */
coli_metal_matmul((ColiMetalTensor**)&((W*)w)->metal, y, x, w->f, NULL, 0, S, w->I, w->O, 0);
return;
}
if(w->fmt==1){
coli_metal_matmul((ColiMetalTensor**)&((W*)w)->metal, y, x, w->q8, w->s, 1, S, w->I, w->O, 0);
return;
}
if(w->fmt==4){
coli_metal_matmul((ColiMetalTensor**)&((W*)w)->metal, y, x, w->q4, w->s, 4, S, w->I, w->O, w->gs);
return;
}
}
#endif
#ifdef COLI_VULKAN
if(g_k3_vk&&S==1&&w->vk&&!g_k3_dho&&
coli_vk_matmul((ColiVkTensor**)&((W*)w)->vk,y,x,NULL,NULL,k3_vk_fmt(w),1,w->I,w->O,w->gs))
return;
/* device only: every row count, from the main thread (one command buffer) */
if(g_k3_dho&&g_k3_vk&&w->vk&&S>=1&&S<=65535&&!k3_in_par()&&
coli_vk_matmul((ColiVkTensor**)&((W*)w)->vk,y,x,NULL,NULL,k3_vk_fmt(w),S,w->I,w->O,w->gs))
return;
if(w->vk_gone) k3_w_reload((W*)w); /* the CPU needs a matrix the device held alone (a lost device) */
#endif
if(w->fmt==0) matmul(y,x,w->f,S,w->I,w->O);
else if(w->fmt==1) matmul_q(y,x,w->q8,w->s,S,w->I,w->O);
else if(w->fmt==4) matmul_i4_grouped(y,x,w->q4,w->s,S,w->I,w->O,w->gs);
else { fprintf(stderr,"w_matmul: bad fmt %d\n",w->fmt); exit(1); }
}
/* f32 GEMM with Metal dispatch: y[S,O] = x[S,I] @ W[O,I]^T */
static void k3_matmul_f32(void *tens, float *y, const float *x, const float *W, int S, int I, int O){
#ifdef COLI_METAL
if(g_k3_metal && coli_metal_available()){
coli_metal_matmul((ColiMetalTensor**)tens, y, x, W, NULL, 0, S, I, O, 0);
return;
}
#endif
matmul(y,x,W,S,I,O);
}
/* acc[0..I) += coef * row r (MLA absorb builds q_abs from kv_b rows) */
static void w_addrow(const W *w, int r, float coef, float *acc){
int I=w->I;
if(w->fmt==0){ const float *p=w->f+(int64_t)r*I; for(int i=0;i<I;i++) acc[i]+=coef*p[i]; }
else if(w->fmt==1){ const int8_t *p=w->q8+(int64_t)r*I; float s=w->s[r]*coef;
for(int i=0;i<I;i++) acc[i]+=s*p[i]; }
else { int rb=(I+1)/2, ng=(I+w->gs-1)/w->gs; const uint8_t *p=w->q4+(int64_t)r*rb;
const float *scl=w->s+(int64_t)r*ng;
for(int g=0;g*w->gs<I;g++){ float s=scl[g]*coef; int e=(g+1)*w->gs; if(e>I)e=I;
for(int i=g*w->gs;i<e;i+=2){ uint8_t b=p[i>>1];
acc[i]+=s*(float)((int)(b&0xF)-8);
if(i+1<e) acc[i+1]+=s*(float)((int)(b>>4)-8); } } }
}
static float w_rowdot(const W *w, int r, const float *x){
int I=w->I; float a=0;
if(w->fmt==0){ const float *p=w->f+(int64_t)r*I; for(int i=0;i<I;i++) a+=x[i]*p[i]; return a; }
if(w->fmt==1){ const int8_t *p=w->q8+(int64_t)r*I; for(int i=0;i<I;i++) a+=x[i]*p[i]; return a*w->s[r]; }
{ int rb=(I+1)/2, ng=(I+w->gs-1)/w->gs; const uint8_t *p=w->q4+(int64_t)r*rb;
const float *scl=w->s+(int64_t)r*ng;
for(int g=0;g*w->gs<I;g++){ float ga=0; int e=(g+1)*w->gs; if(e>I)e=I;
for(int i=g*w->gs;i<e;i+=2){ uint8_t b=p[i>>1];
ga+=x[i]*(float)((int)(b&0xF)-8);
if(i+1<e) ga+=x[i+1]*(float)((int)(b>>4)-8); }
a+=ga*scl[g]; } }
return a;
}
#define QCHUNK 1024 /* rows per load-quantize pass */
static int g_bits_env=0; /* K3_BITS explicitly set: enables the
* int8-container -> int4 load downcast */
static int g_k3_mmap=0; /* prepared U8/F32 tensors stay file-backed */
static uint64_t g_k3_mmap_bytes=0, g_k3_mmap_views=0;
#ifdef COLI_CUDA
static int g_k3_cuda=0; /* K3_CUDA=1: MXFP4 routed experts on CUDA at decode */
#endif
static int g_k3_direct=-1; /* K3_DIRECT: O_DIRECT expert reads */
static int g_k3_idot=1; /* K3_IDOT: int8-activation expert matmuls */
static int g_k3_pipe=1; /* K3_PIPE: overlap loads with compute */
static float g_k3_topp=0.f; /* K3_TOPP: routed-expert top-p pruning */
enum { K3_CKPT_MAX = 8 };
static size_t k3_ckpt_blob_floats(const Model *m); /* def. with the ckpt block */
static double k3_ckpt_reserve_gb(const Model *m, int slots);
static int g_k3_ckpt_slots=0; /* COLI_K3_CKPT: recurrent-state checkpoint
* slots. STRICTLY OPT-IN, 0 = off (default,
* engine behaves exactly as before). */
static const char *g_k3_ckpt_dir=NULL; /* COLI_K3_CKPT_DIR: park the photos in
* files instead of RAM (low-RAM machines:
* ~0.4s of sequential I/O per restore
* against minutes of avoided re-prefill). */
static int k3_mmap_backend_allowed(int mmap_enabled, int vk_enabled, int cuda_enabled){
return !mmap_enabled || (!vk_enabled && !cuda_enabled);
}
static void w_map_prepared(Model *m, W *w, const char *name, const char *scale_name,
st_tensor *t, st_tensor *ts){
if(ts->dtype!=2){
fprintf(stderr,"K3_MMAP=1 requires F32 scale sidecars; %s is %s -- refusing\n",
scale_name,st_dtype_name(ts->dtype)); exit(1);
}
if(st_map_raw(&m->S,name,&w->data_map)!=0){
fprintf(stderr,"K3_MMAP=1 could not map %s: %s -- refusing\n",name,strerror(errno)); exit(1);
}
if(st_map_raw(&m->S,scale_name,&w->scale_map)!=0){
int saved=errno; st_unmap_raw(&w->data_map);
fprintf(stderr,"K3_MMAP=1 could not map %s: %s -- refusing\n",scale_name,strerror(saved)); exit(1);
}
if(((uintptr_t)w->scale_map.data%_Alignof(float))!=0){
st_unmap_raw(&w->scale_map); st_unmap_raw(&w->data_map);
fprintf(stderr,"K3_MMAP=1 scale sidecar %s is not float-aligned -- refusing\n",scale_name); exit(1);
}
w->mapped=1;
w->s=(float*)w->scale_map.data;
g_k3_mmap_bytes+=(uint64_t)t->nbytes+(uint64_t)ts->nbytes;
g_k3_mmap_views+=2;
}
static void w_release_host(W *w){
if(!w) return;
if(w->mapped){ st_unmap_raw(&w->scale_map); st_unmap_raw(&w->data_map); }
else { free(w->f); free(w->q8); free(w->q4); free(w->s); }
#ifdef COLI_VULKAN
free(w->vk_name);
#endif
memset(w,0,sizeof(*w));
}
static void w_load(Model *m, W *w, const char *name, int O, int I, int bits){
char nm[512]; snprintf(nm,sizeof(nm),"%s%s",m->pfx,name);
st_tensor *t=st_find(&m->S,nm);
if(!t) st_die_missing(&m->S,nm);
memset(w,0,sizeof(*w)); w->O=O; w->I=I;
#ifdef COLI_VULKAN
w->vk_name=strdup(name); w->vk_bits=bits; /* to read it back as it is read now (k3_w_reload) */
if(!w->vk_name){fprintf(stderr,"OOM weight name\n");exit(1);}
#endif
if(t->dtype==3){
/* repacked container (tools/k3_repack.py): pre-quantized U8 + .qs f32
* scales — no load-time quantization, K3_BITS is ignored for these
* (except the explicit =4 downcast below) */
char qn[560]; snprintf(qn,sizeof(qn),"%s.qs",nm);
st_tensor *ts=st_find(&m->S,qn);
if(!ts){ fprintf(stderr,"%s: quantized (U8) but no %s scale sidecar\n",nm,qn); exit(1); }
if(t->nbytes==(int64_t)O*I && ts->numel==O){ /* int8 per-row */
if(g_bits_env && bits==4 && I%64==0){
if(g_k3_mmap){
fprintf(stderr,"K3_MMAP=1 cannot combine with load-time int8-to-int4 conversion for %s; use a prepared int4 container or unset K3_BITS -- refusing\n",nm);
exit(1);
}
/* EXPLICIT K3_BITS=4 on an int8 container: downcast to int4-g64
* at load. Halves resident RAM (the 62 GB box cannot hold the
* 93-layer non-expert set at int8 next to a desktop session);
* the int8 grid is 16x finer than int4, so the double-quant
* noise ~ the direct-int4 noise. Unset K3_BITS keeps the
* container's own bits — the default is untouched. */
int gs=64, rb=I/2, ng=I/gs;
int8_t *q8=malloc((size_t)O*I); float *s8=falloc(O);
if(!q8){fprintf(stderr,"OOM int8 tmp %s\n",nm);exit(1);}
st_read_raw(&m->S,nm,q8,1); st_read_f32(&m->S,qn,s8,0);
w->fmt=4; w->gs=gs;
w->q4=malloc((int64_t)O*rb); w->s=falloc((int64_t)O*ng);
if(!w->q4){fprintf(stderr,"OOM int4 %s\n",nm);exit(1);}
for(int r=0;r<O;r++){
const int8_t *src=q8+(int64_t)r*I; float sc8=s8[r];
uint8_t *dst=w->q4+(int64_t)r*rb; float *scl=w->s+(int64_t)r*ng;
for(int g=0;g<ng;g++){ const int8_t *gp=src+g*gs;
int am=0; for(int i=0;i<gs;i++){ int a=gp[i]<0?-gp[i]:gp[i]; if(a>am)am=a; }
float s=am*sc8/7.f; if(s<1e-20f)s=1e-20f; scl[g]=s; float inv=sc8/s;
for(int i=0;i<gs;i+=2){
int v0=(int)lrintf(gp[i]*inv); if(v0>7)v0=7; if(v0<-8)v0=-8;
int v1=(int)lrintf(gp[i+1]*inv); if(v1>7)v1=7; if(v1<-8)v1=-8;
dst[(g*gs+i)>>1]=(uint8_t)((v0+8)|((v1+8)<<4)); } } }
free(q8); free(s8);
return;
}
w->fmt=1;
if(g_k3_mmap){
w_map_prepared(m,w,nm,qn,t,ts); w->q8=(int8_t*)w->data_map.data;
return;
}
w->q8=malloc((size_t)O*I);
if(!w->q8){fprintf(stderr,"OOM int8 %s\n",nm);exit(1);}
st_read_raw(&m->S,nm,w->q8,1);
w->s=falloc(O); st_read_f32(&m->S,qn,w->s,0);
} else if(I%64==0 && t->nbytes==(int64_t)O*(I/2) && ts->numel==(int64_t)O*(I/64)){
w->fmt=4; w->gs=64; /* int4-g64 */
if(g_k3_mmap){
w_map_prepared(m,w,nm,qn,t,ts); w->q4=(uint8_t*)w->data_map.data;
return;
}
w->q4=malloc((size_t)O*(I/2));
if(!w->q4){fprintf(stderr,"OOM int4 %s\n",nm);exit(1);}
st_read_raw(&m->S,nm,w->q4,1);
w->s=falloc((int64_t)O*(I/64)); st_read_f32(&m->S,qn,w->s,0);
} else {
fprintf(stderr,"%s: U8 tensor is %lld bytes / %lld scales — matches neither int8 [%d,%d] nor int4-g64, refusing (untrusted container)\n",
nm,(long long)t->nbytes,(long long)ts->numel,O,I); exit(1);
}
return;
}
if(g_k3_mmap){
fprintf(stderr,"K3_MMAP=1 requires a fully prepared U8 + F32-sidecar container; %s is %s and would need load-time conversion -- refusing\n",
nm,st_dtype_name(t->dtype)); exit(1);
}
if(t->numel!=(int64_t)O*I){ fprintf(stderr,"%s: numel %lld != %dx%d\n",nm,(long long)t->numel,O,I); exit(1); }
if(bits>=32){ w->fmt=0; w->f=falloc((int64_t)O*I); st_read_f32(&m->S,nm,w->f,0); return; }
int gs=64;
if(bits<=4 && I%gs){ bits=8; } /* int4-g64 wants I%64==0; fall back */
float *scr=falloc((int64_t)QCHUNK*I);
if(bits>4){ w->fmt=1; w->q8=malloc((int64_t)O*I); w->s=falloc(O);
if(!w->q8){fprintf(stderr,"OOM int8 %s\n",nm);exit(1);}
for(int r0=0;r0<O;r0+=QCHUNK){ int n=O-r0<QCHUNK?O-r0:QCHUNK;
st_read_slice_f32(&m->S,nm,(int64_t)r0*I,(int64_t)n*I,scr,1);
for(int r=0;r<n;r++){ const float *src=scr+(int64_t)r*I;
float am=0; for(int i=0;i<I;i++){ float a=fabsf(src[i]); if(a>am)am=a; }
float s=am/127.f; if(s<1e-20f)s=1e-20f; w->s[r0+r]=s; float inv=1.f/s;
int8_t *dst=w->q8+(int64_t)(r0+r)*I;
for(int i=0;i<I;i++){ int v=(int)lrintf(src[i]*inv); if(v>127)v=127; if(v<-127)v=-127; dst[i]=(int8_t)v; } } }
} else { w->fmt=4; w->gs=gs; int rb=I/2, ng=I/gs;
w->q4=malloc((int64_t)O*rb); w->s=falloc((int64_t)O*ng);
if(!w->q4){fprintf(stderr,"OOM int4 %s\n",nm);exit(1);}
for(int r0=0;r0<O;r0+=QCHUNK){ int n=O-r0<QCHUNK?O-r0:QCHUNK;
st_read_slice_f32(&m->S,nm,(int64_t)r0*I,(int64_t)n*I,scr,1);
for(int r=0;r<n;r++){ const float *src=scr+(int64_t)r*I;
uint8_t *dst=w->q4+(int64_t)(r0+r)*rb; float *scl=w->s+(int64_t)(r0+r)*ng;
for(int g=0;g<ng;g++){ const float *gp=src+g*gs;
float am=0; for(int i=0;i<gs;i++){ float a=fabsf(gp[i]); if(a>am)am=a; }
float s=am/7.f; if(s<1e-20f)s=1e-20f; scl[g]=s; float inv=1.f/s;
for(int i=0;i<gs;i+=2){
int v0=(int)lrintf(gp[i]*inv); if(v0>7)v0=7; if(v0<-8)v0=-8;
int v1=(int)lrintf(gp[i+1]*inv); if(v1>7)v1=7; if(v1<-8)v1=-8;
dst[(g*gs+i)>>1]=(uint8_t)((v0+8)|((v1+8)<<4)); } } } }
}
free(scr);
}
static float *f32_load(Model *m, const char *name, int64_t want){
char nm[512]; snprintf(nm,sizeof(nm),"%s%s",m->pfx,name);
st_tensor *t=st_find(&m->S,nm);
if(!t) st_die_missing(&m->S,nm);
if(want>0 && t->numel!=want){ fprintf(stderr,"%s: numel %lld != %lld\n",nm,(long long)t->numel,(long long)want); exit(1); }
float *p=falloc(t->numel); st_read_f32(&m->S,nm,p,0); return p;
}
#ifdef COLI_VULKAN
/* ---- the dense matrices on the device only (COLI_VK_DENSE_HOST) -----------------
* Decided once the device is open and before the expert cache is sized (k3_dho_decide),
* so the cache counts the dense weights out of RAM; placed in k3_vk_tier_start: every W
* goes up into its own vk (the chain adopts those copies), then its host copy is
* dropped, and the f32 matrices only the chain multiplies on the device (the routers,
* the KDA decay and beta projections) are dropped once the chain holds them. The CPU
* reads a matrix back from disk when it needs it after all: w_matmul after a lost
* device, the CPU MLA path's kv_b rows, the CPU KDA and router matmuls (k3_f32_need). */
static Model *g_k3_dho_model;
static size_t g_k3_dho_bytes; /* what will be dropped: RAM the expert cache may count on */
static pthread_mutex_t g_k3_dho_mx=PTHREAD_MUTEX_INITIALIZER;
static size_t k3_w_host_bytes(const W *w){
if(w->fmt==0) return w->f?(size_t)w->O*w->I*4:0;
if(w->fmt==1) return w->q8?(size_t)w->O*w->I+(size_t)w->O*4:0;
if(w->fmt==4) return w->q4?(size_t)w->O*((w->I+1)/2)+(size_t)w->O*((w->I+w->gs-1)/w->gs)*4:0;
return 0;
}
static void k3_w_reload(W *w){
pthread_mutex_lock(&g_k3_dho_mx);
if(w->vk_gone){
Model *m=g_k3_dho_model;
if(!m||!w->vk_name){fprintf(stderr,"[VK] kimi_k3: a dense matrix the device held alone cannot be read back\n");exit(1);}
W t; w_load(m,&t,w->vk_name,w->O,w->I,w->vk_bits);
if(t.fmt!=w->fmt||t.gs!=w->gs){fprintf(stderr,"[VK] kimi_k3: %s came back in another form\n",w->vk_name);exit(1);}
w->f=t.f; w->q8=t.q8; w->q4=t.q4; w->s=t.s;
free(t.vk_name);
w->vk_gone=0;
coli_vk_dense_host_reloaded(k3_w_host_bytes(w));
}
pthread_mutex_unlock(&g_k3_dho_mx);
}
/* The f32 matrices the chain holds on the device: where each lives, and its name. */
typedef struct { float **p; char name[160]; int64_t n; } K3F32Home;
static K3F32Home *g_k3_f32; static int g_k3_f32_n;
/* A dropped f32 matrix the CPU needs: read it back (NULL is "dropped": loaded ones never are). */
static void k3_f32_need(float **p){
if(*p) return;
pthread_mutex_lock(&g_k3_dho_mx);
if(!*p){
for(int i=0;i<g_k3_f32_n;i++) if(g_k3_f32[i].p==p){
*p=f32_load(g_k3_dho_model,g_k3_f32[i].name,g_k3_f32[i].n);
coli_vk_dense_host_reloaded((size_t)g_k3_f32[i].n*4);
break;
}
if(!*p){fprintf(stderr,"[VK] kimi_k3: an f32 matrix the device held alone cannot be read back\n");exit(1);}
}
pthread_mutex_unlock(&g_k3_dho_mx);
}
#define K3_F32_NEED(p) k3_f32_need(&(p))
/* y = x W^T on the device with the chain's copy of an f32 matrix (the device only: the CPU
* may hold none); 0 = the CPU multiplies, reading the matrix back first if it must. */
static int k3_f32_dev(void *t, float *y, const float *x, int S, int I, int O){
return g_k3_dho&&g_k3_vk&&t&&S>=1&&S<=65535&&!k3_in_par()&&
coli_vk_matmul((ColiVkTensor**)&t,y,x,NULL,NULL,10,S,I,O,0);
}
#define K3_F32_DEV(t,y,x,S,I,O) k3_f32_dev(t,y,x,S,I,O)
/* Every W of the model, in a fixed order (with a partial chain, its layers' and the head
* only when it goes to the device as well). */
static void k3_dho_each(Model *m, void (*f)(W *, size_t *, int *), size_t *b, int *n){
for(int i=0;i<m->c.n_layers&&i<g_k3_dho_layers;i++){
Layer *l=&m->L[i];
if(l->kda){ Kda *a=&l->a; W *ws[]={&a->q,&a->k,&a->v,&a->g,&a->o}; for(size_t k=0;k<5;k++) f(ws[k],b,n); }
else { Mla *a=&l->m; W *ws[]={&a->qa,&a->qb,&a->kva,&a->kvb,&a->o,&a->g,&a->wk,&a->wq,&a->wp}; for(size_t k=0;k<9;k++) f(ws[k],b,n); }
if(l->sparse){ Moe *o=&l->moe; W *ws[]={&o->lat_down,&o->lat_up,&o->sh_gate,&o->sh_up,&o->sh_down}; for(size_t k=0;k<5;k++) f(ws[k],b,n); }
else { W *ws[]={&l->d_gate,&l->d_up,&l->d_down}; for(size_t k=0;k<3;k++) f(ws[k],b,n); }
}
if(m->has_head&&g_k3_dho_head) f(&m->lm_head,b,n);
}
static void k3_dho_count(W *w, size_t *b, int *n){ size_t x=k3_w_host_bytes(w); if(x&&!w->mapped&&k3_vk_fmt(w)>0){ *b+=x; (*n)++; } }
/* The f32 matrices the chain multiplies on the device: the routers, the KDA decay pair and beta. */
static size_t k3_dho_f32_bytes(const Model *m){
const Cfg *c=&m->c; size_t b=0;
for(int i=0;i<c->n_layers&&i<g_k3_dho_layers;i++){
const Layer *l=&m->L[i];
if(l->sparse&&l->moe.router) b+=(size_t)c->n_experts*c->hidden*4;
if(l->kda&&l->a.fa) b+=((size_t)c->kda_hd*c->hidden+(size_t)c->kda_proj*c->kda_hd+(size_t)c->kda_heads*c->hidden)*4;
}
return b;
}
/* Once the device is open and before the expert cache is sized: the decision, with the
* chain's own decision asked silently (k3c_start prints it later). The cache counts the
* W matrices out of RAM; the f32 ones only leave once the chain has them, so it does not
* count on those. */
static int k3c_fit_now(Model *m, int tier_on, int cuda_on); /* kimi_k3_chain.h */
static void k3_dho_decide(Model *m, int tier_on, int cuda_on){
if(!g_k3_vk) return;
int chain=!cuda_on&&!k3_dsa_indexer_on()&&coli_vk_chain_decide(NULL,tier_on,COLI_VK_CHAIN_UNMEASURED)!=COLI_VK_CHAIN_OFF;
int dense_on=coli_vk_dense()||chain;
if(k3c_fit_now(m,tier_on,cuda_on)){ /* a partial chain: its layers, and the head with the tail */
g_k3_dho_layers=g_k3_fit.n; g_k3_dho_head=g_k3_fit.n==m->c.n_layers&&g_k3_fit.tail;
chain=g_k3_fit.n>0; dense_on=chain||(coli_vk_dense()&&!g_k3_partial);
}
size_t b=0; int n=0;
k3_dho_each(m,k3_dho_count,&b,&n);
int on=coli_vk_dense_host_decide("kimi_k3",dense_on,b+(chain?k3_dho_f32_bytes(m):0));
if(g_k3_fit.L>0) coli_vk_dense_host_layers(g_k3_fit.n,m->c.n_layers);
if(!on) return;
g_k3_dho=1; g_k3_dho_model=m; g_k3_dho_bytes=b;
}
#else
#define K3_F32_NEED(p) ((void)0)
#define K3_F32_DEV(t,y,x,S,I,O) 0
#endif
/* ---------- config ---------- */
static double req_num(jval *r, const char *k){
jval *v=json_get(r,k);
if(!v||v->t!=J_NUM){ fprintf(stderr,"config.json: missing or non-numeric \"%s\"\n",k); exit(1); }
return v->num;
}
static void load_cfg(Cfg *c, const char *snap){
char path[2048]; snprintf(path,sizeof(path),"%s/config.json",snap);
long n; char *buf;
{ FILE *f=fopen(path,"rb"); if(!f){perror(path);exit(1);}
fseek(f,0,SEEK_END); n=ftell(f); fseek(f,0,SEEK_SET);
if(n<0||n>(64L<<20)){ fprintf(stderr,"%s: bad size\n",path); exit(1); }
buf=malloc((size_t)n+1); if(!buf){fprintf(stderr,"OOM cfg\n");exit(1);}
if(fread(buf,1,(size_t)n,f)!=(size_t)n){ fprintf(stderr,"%s: short read\n",path); exit(1); }
buf[n]=0; fclose(f); }
char *arena=NULL; jval *root=json_parse(buf,&arena);
jval *tc=json_get(root,"text_config"); if(!tc||tc->t!=J_OBJ) tc=root;
memset(c,0,sizeof(*c));
c->hidden =(int)req_num(tc,"hidden_size");
c->n_layers =(int)req_num(tc,"num_hidden_layers");
c->vocab =(int)req_num(tc,"vocab_size");
c->first_dense =(int)req_num(tc,"first_k_dense_replace");
c->dense_inter =(int)req_num(tc,"intermediate_size");
c->n_heads =(int)req_num(tc,"num_attention_heads");
c->q_lora =(int)req_num(tc,"q_lora_rank");
c->kv_lora =(int)req_num(tc,"kv_lora_rank");
c->qk_nope =(int)req_num(tc,"qk_nope_head_dim");
c->qk_rope =(int)req_num(tc,"qk_rope_head_dim");
c->v_head =(int)req_num(tc,"v_head_dim");
c->n_experts =(int)req_num(tc,"num_experts");
c->topk =(int)req_num(tc,"num_experts_per_token");
c->moe_inter =(int)req_num(tc,"moe_intermediate_size");
c->latent =(int)req_num(tc,"routed_expert_hidden_size");
c->n_shared =(int)req_num(tc,"num_shared_experts");
c->res_bs =(int)req_num(tc,"attn_res_block_size");
c->situ_b1 =(float)req_num(tc,"activation_situ_beta");
c->situ_b2 =(float)req_num(tc,"activation_situ_linear_beta");
jval *ep=json_get(tc,"rms_norm_eps"); c->eps=ep?(float)ep->num:1e-5f;
jval *rp=json_get(tc,"rope_parameters");
if(rp&&rp->t==J_OBJ){ jval *th=json_get(rp,"rope_theta"); c->theta=th?(float)th->num:10000.f; }
else c->theta=10000.f;
c->qk_head=c->qk_nope+c->qk_rope;
c->attn_scale=1.f/sqrtf((float)c->qk_head);
jval *la=json_get(tc,"linear_attn_config");
if(!la||la->t!=J_OBJ){ fprintf(stderr,"config.json: missing linear_attn_config\n"); exit(1); }
c->kda_heads=(int)req_num(la,"num_heads");
c->kda_hd =(int)req_num(la,"head_dim");
c->conv_k =(int)req_num(la,"short_conv_kernel_size");
jval *lb=json_get(la,"gate_lower_bound"); c->gate_lb=lb?(float)lb->num:-5.f;
c->kda_proj=c->kda_heads*c->kda_hd;
if(c->hidden<1||c->hidden>65536||c->n_layers<1||c->n_layers>128||
c->n_experts<1||c->n_experts>4096||c->topk<1||c->topk>64||c->topk>c->n_experts||
c->vocab<1||c->vocab>(1<<22)||c->kda_proj<1||c->kda_proj>(1<<20)||
c->conv_k<1||c->conv_k>8||c->latent<32||c->latent%32||c->moe_inter%32||
c->res_bs<1||c->kda_hd>512||c->kv_lora>4096){
fprintf(stderr,"config.json: dimension out of range\n"); exit(1); }
jval *kl=json_get(la,"kda_layers");
if(!kl||kl->t!=J_ARR){ fprintf(stderr,"config.json: missing kda_layers\n"); exit(1); }
for(int i=0;i<kl->len;i++){ int v=(int)kl->kids[i]->num; /* 1-indexed */
if(v>=1&&v<=c->n_layers) c->is_kda[v-1]=1; }
/* DSA indexer (optional — if absent in config, DSA is disabled) */
{ jval *jn=json_get(tc,"index_hd"); c->index_hd = jn&&jn->t==J_NUM ? (int)jn->num : 0; }
{ jval *jn=json_get(tc,"index_nh"); c->index_nh = jn&&jn->t==J_NUM ? (int)jn->num : 0; }
{ jval *jn=json_get(tc,"index_topk"); c->index_topk = jn&&jn->t==J_NUM ? (int)jn->num : 0; }
if(c->index_hd > 0){
/* Default: all MLA layers are "full" DSA layers (compute own Ic + top-K) */
for(int i=0;i<c->n_layers;i++) if(!c->is_kda[i]) c->idx_type[i]=1;
jval *il=json_get(tc,"index_layers");
if(il && il->t==J_ARR){
/* Override: explicit list of 1-indexed full DSA layers */
memset(c->idx_type, 0, sizeof(c->idx_type));
for(int i=0;i<il->len;i++){ int v=(int)il->kids[i]->num;
if(v>=1&&v<=c->n_layers) c->idx_type[v-1]=1; }
}
}
/* the DSA indexer ropes the first qk_rope_head_dim floats of each of its index_hd-long
* rows (dsa_rope) */
if(c->index_hd > 0 && c->index_hd < c->qk_rope){
fprintf(stderr,"config.json: index_hd=%d is shorter than qk_rope_head_dim=%d, which the "
"indexer ropes in each of its rows\n",c->index_hd,c->qk_rope); exit(1); }
jval *b=json_get(root,"bos_token_id"); if(!b) b=json_get(tc,"bos_token_id");
c->bos = b&&b->t==J_NUM ? (int)b->num : -1;
jval *e=json_get(root,"eos_token_id"); if(!e) e=json_get(tc,"eos_token_id");
if(e&&e->t==J_NUM) c->eos[c->n_eos++]=(int)e->num;
else if(e&&e->t==J_ARR) for(int i=0;i<e->len&&c->n_eos<8;i++) c->eos[c->n_eos++]=(int)e->kids[i]->num;
free(buf); (void)arena;
}
/* ---------- init ---------- */
static void expert_table_init(Model *m){
Cfg *c=&m->c;
m->e_w1p=(int64_t)c->moe_inter*(c->latent/2); m->e_w1s=(int64_t)c->moe_inter*(c->latent/32);
m->e_w2p=(int64_t)c->latent*(c->moe_inter/2); m->e_w2s=(int64_t)c->latent*(c->moe_inter/32);
m->e_slot=2*(m->e_w1p+m->e_w1s)+m->e_w2p+m->e_w2s;
m->eref=calloc((size_t)c->n_layers*c->n_experts,sizeof(ERef));
if(!m->eref){fprintf(stderr,"OOM expert table\n");exit(1);}
const char *mat[3]={"w1","w2","w3"};
int64_t want[6]={m->e_w1p,m->e_w1s,m->e_w2p,m->e_w2s,m->e_w1p,m->e_w1s};
int missing=0;
for(int li=0;li<c->n_layers;li++){
if(!m->L[li].sparse) continue;
for(int e2=0;e2<c->n_experts;e2++){
ERef *er=&m->eref[(int64_t)li*c->n_experts+e2];
for(int k=0;k<6;k++){
char nm[512];
snprintf(nm,sizeof(nm),"%smodel.layers.%d.block_sparse_moe.experts.%d.%s.weight_%s",
m->pfx,li,e2,mat[k/2],(k&1)?"scale":"packed");
st_tensor *t=st_find(&m->S,nm);
if(!t){ missing++; er->fd[k]=-1; continue; }
if(t->nbytes!=want[k]){ fprintf(stderr,"%s: %lld bytes, expected %lld — refusing (untrusted container)\n",
nm,(long long)t->nbytes,(long long)want[k]); exit(1); }
er->fd[k]=t->fd; er->off[k]=t->off;
}
/* HF shards store the six tensors back-to-back (measured: 0 gaps
* across a whole layer) — collapse the load to ONE pread when so */
er->contig=1;
for(int k=0;k<5;k++)
if(er->fd[k]!=er->fd[k+1]||er->off[k]+want[k]!=er->off[k+1]) er->contig=0;
}
}
if(missing) fprintf(stderr,"[K3] WARNING: %d expert tensors missing (incomplete download?) — touching one aborts\n",missing);
}
static void model_init_range(Model *m, const char *snap, int layer_begin,
int layer_end, int truncate_model,
int load_boundaries, int allocate_state,
int expert_cap_override, int context_hint,
int segment_mode){
memset(m,0,sizeof(*m));
load_cfg(&m->c,snap);
Cfg *c=&m->c;
if(truncate_model&&layer_end>0&&layer_end<c->n_layers) c->n_layers=layer_end;
if(layer_end==0) layer_end=c->n_layers;
if(layer_begin<0||layer_end>c->n_layers||layer_begin>=layer_end){
fprintf(stderr,"invalid Kimi K3 layer range [%d,%d) for %d layers\n",
layer_begin,layer_end,c->n_layers); exit(1); }
st_init_multi(&m->S,snap,getenv("K3_DIRS")); /* K3_DIRS: extra shard dirs (multi-drive split) */
m->pfx[0]=0; /* probe a layer-0 tensor: embed/head live in one of the LAST shards */
if(!st_has(&m->S,"model.layers.0.input_layernorm.weight")&&
st_has(&m->S,"language_model.model.layers.0.input_layernorm.weight"))
snprintf(m->pfx,sizeof(m->pfx),"language_model.");
if((c->n_layers+c->res_bs-1)/c->res_bs+1>16){ fprintf(stderr,"attn_res: too many blocks\n"); exit(1); }
g_bits_env = getenv("K3_BITS")!=NULL;
g_k3_mmap = getenv("K3_MMAP")?atoi(getenv("K3_MMAP")):0;
if(g_k3_mmap){
int vk_requested=0, cuda_requested=0;
#ifdef COLI_VULKAN
vk_requested=k3_vk_requested();
#endif
#ifdef COLI_CUDA
cuda_requested=getenv("K3_CUDA")&&atoi(getenv("K3_CUDA"));
#endif
if(!k3_mmap_backend_allowed(1,vk_requested,cuda_requested)){
fprintf(stderr,"K3_MMAP=1 is CPU-only; unset COLI_VULKAN (or set K3_VK=0) and K3_CUDA=0 -- refusing before model load\n");
exit(1);
}
fprintf(stderr,"[K3-MMAP] prepared tensor mapping enabled (CPU-only, no conversion fallback)\n");
}
#ifdef COLI_CUDA
g_k3_cuda = getenv("K3_CUDA") ? atoi(getenv("K3_CUDA")) : 0;
if(g_k3_cuda){
int dev0 = 0;
if(!coli_cuda_init(&dev0, 1)){
fprintf(stderr,"[K3-CUDA] device unavailable -- experts stay on CPU\n");
g_k3_cuda = 0;
} else fprintf(stderr,"[K3-CUDA] MXFP4 routed experts on device 0 (decode only)\n");
}
#endif
g_k3_direct = getenv("K3_DIRECT")?atoi(getenv("K3_DIRECT")):1;
g_k3_idot = getenv("K3_IDOT")?atoi(getenv("K3_IDOT")):1;
{ /* Recurrent-state checkpoints are strictly OPT-IN: unset or 0 keeps
* the engine byte-identical to before the feature existed. */
const char *e=segment_mode?NULL:getenv("COLI_K3_CKPT");
g_k3_ckpt_slots=e?atoi(e):0;
if(g_k3_ckpt_slots<0) g_k3_ckpt_slots=0;
if(g_k3_ckpt_slots>K3_CKPT_MAX) g_k3_ckpt_slots=K3_CKPT_MAX;
g_k3_ckpt_dir=segment_mode?NULL:getenv("COLI_K3_CKPT_DIR");
if(g_k3_ckpt_dir&&!*g_k3_ckpt_dir) g_k3_ckpt_dir=NULL;
/* a directory alone is a clear enough request: default to 2 slots
* there, while an explicit COLI_K3_CKPT (0 included) always wins */
if(g_k3_ckpt_dir&&!e) g_k3_ckpt_slots=2; }
/* The legacy loader pool is process-global and keeps a Model pointer.
* Segment hosts may own several engines, so their demand loads stay
* synchronous until the pool becomes engine-owned. */
g_k3_pipe = segment_mode ? 0
: (getenv("K3_PIPE")?atoi(getenv("K3_PIPE")):1);
/* Segment numeric classes are exact by default: do not inherit the
* standalone quality lever from a donor's environment. */
g_k3_topp = segment_mode ? 0.f
: (getenv("K3_TOPP")?(float)atof(getenv("K3_TOPP")):0.f);
if(g_k3_topp>0.f)
fprintf(stderr,"[K3] TOPP=%.2f: routed experts pruned to cumulative weight (quality lever — A/B with K3_LOGITS)\n",g_k3_topp);
int bits = getenv("K3_BITS")?atoi(getenv("K3_BITS")):4;
int mbits = getenv("K3_MLA_BITS")?atoi(getenv("K3_MLA_BITS")):8;
int hbits = getenv("K3_HEAD_BITS")?atoi(getenv("K3_HEAD_BITS")):8;
double t0=now_s();
m->L=calloc(c->n_layers,sizeof(Layer));
m->kstate=calloc(c->n_layers,sizeof(float*));
m->cwq=calloc(c->n_layers,sizeof(float*));
m->cwk=calloc(c->n_layers,sizeof(float*));
m->cwv=calloc(c->n_layers,sizeof(float*));
char nm[512];
#define NM(...) (snprintf(nm,sizeof(nm),__VA_ARGS__),nm)
for(int i=layer_begin;i<layer_end;i++){
Layer *l=&m->L[i];
l->kda=c->is_kda[i];
l->sparse=(i>=c->first_dense);
l->in_ln =f32_load(m,NM("model.layers.%d.input_layernorm.weight",i),c->hidden);
l->post_ln=f32_load(m,NM("model.layers.%d.post_attention_layernorm.weight",i),c->hidden);
/* AttnRes score weight = res_norm.weight * res_proj.weight (elementwise) */
{ float *rn=f32_load(m,NM("model.layers.%d.self_attention_res_norm.weight",i),c->hidden);
float *rp=f32_load(m,NM("model.layers.%d.self_attention_res_proj.weight",i),c->hidden);
l->attn_sw=falloc(c->hidden); for(int d=0;d<c->hidden;d++) l->attn_sw[d]=rn[d]*rp[d];
free(rn); free(rp);
rn=f32_load(m,NM("model.layers.%d.mlp_res_norm.weight",i),c->hidden);
rp=f32_load(m,NM("model.layers.%d.mlp_res_proj.weight",i),c->hidden);
l->mlp_sw=falloc(c->hidden); for(int d=0;d<c->hidden;d++) l->mlp_sw[d]=rn[d]*rp[d];
free(rn); free(rp); }
if(l->kda){
Kda *a=&l->a; int P=c->kda_proj;
w_load(m,&a->q,NM("model.layers.%d.self_attn.q_proj.weight",i),P,c->hidden,bits);
w_load(m,&a->k,NM("model.layers.%d.self_attn.k_proj.weight",i),P,c->hidden,bits);
w_load(m,&a->v,NM("model.layers.%d.self_attn.v_proj.weight",i),P,c->hidden,bits);
w_load(m,&a->g,NM("model.layers.%d.self_attn.g_proj.weight",i),P,c->hidden,bits);
w_load(m,&a->o,NM("model.layers.%d.self_attn.o_proj.weight",i),c->hidden,P,bits);
a->conv_q=f32_load(m,NM("model.layers.%d.self_attn.q_conv1d.weight",i),(int64_t)P*c->conv_k);
a->conv_k=f32_load(m,NM("model.layers.%d.self_attn.k_conv1d.weight",i),(int64_t)P*c->conv_k);
a->conv_v=f32_load(m,NM("model.layers.%d.self_attn.v_conv1d.weight",i),(int64_t)P*c->conv_k);
a->fa=f32_load(m,NM("model.layers.%d.self_attn.f_a_proj.weight",i),(int64_t)c->kda_hd*c->hidden);
a->fb=f32_load(m,NM("model.layers.%d.self_attn.f_b_proj.weight",i),(int64_t)P*c->kda_hd);
a->bp=f32_load(m,NM("model.layers.%d.self_attn.b_proj.weight",i),(int64_t)c->kda_heads*c->hidden);
a->dt=f32_load(m,NM("model.layers.%d.self_attn.dt_bias",i),P);
a->onw=f32_load(m,NM("model.layers.%d.self_attn.o_norm.weight",i),c->kda_hd);
{ /* A_log in the checkpoint is [kda_hd] = per-head zero-padded */
char an[512]; snprintf(an,sizeof(an),"%smodel.layers.%d.self_attn.A_log",m->pfx,i);
st_tensor *t=st_find(&m->S,an); if(!t) st_die_missing(&m->S,an);
if(t->numel<c->kda_heads){ fprintf(stderr,"%s: %lld < heads\n",an,(long long)t->numel); exit(1); }
float *al=falloc(t->numel); st_read_f32(&m->S,an,al,0);
a->A=falloc(c->kda_heads);
for(int h=0;h<c->kda_heads;h++) a->A[h]=expf(al[h]);
free(al); }
/* afcalloc: 16 KB-aligned so Metal wraps these zero-copy and GPU state/window
* updates persist across tokens (see afcalloc note). CPU path is unaffected. */
if(allocate_state){
m->kstate[i]=afcalloc((int64_t)c->kda_heads*c->kda_hd*c->kda_hd);
m->cwq[i]=afcalloc((int64_t)P*c->conv_k);
m->cwk[i]=afcalloc((int64_t)P*c->conv_k);
m->cwv[i]=afcalloc((int64_t)P*c->conv_k);
}
} else {
Mla *a=&l->m;
w_load(m,&a->qa,NM("model.layers.%d.self_attn.q_a_proj.weight",i),c->q_lora,c->hidden,mbits);
w_load(m,&a->qb,NM("model.layers.%d.self_attn.q_b_proj.weight",i),c->n_heads*c->qk_head,c->q_lora,mbits);
w_load(m,&a->kva,NM("model.layers.%d.self_attn.kv_a_proj_with_mqa.weight",i),c->kv_lora+c->qk_rope,c->hidden,mbits);
w_load(m,&a->kvb,NM("model.layers.%d.self_attn.kv_b_proj.weight",i),c->n_heads*(c->qk_nope+c->v_head),c->kv_lora,mbits);
w_load(m,&a->o,NM("model.layers.%d.self_attn.o_proj.weight",i),c->hidden,c->n_heads*c->v_head,mbits);
w_load(m,&a->g,NM("model.layers.%d.self_attn.g_proj.weight",i),c->n_heads*c->v_head,c->hidden,mbits);
a->qa_ln =f32_load(m,NM("model.layers.%d.self_attn.q_a_layernorm.weight",i),c->q_lora);
a->kva_ln=f32_load(m,NM("model.layers.%d.self_attn.kv_a_layernorm.weight",i),c->kv_lora);
/* DSA indexer weights – only for "full" layers (idx_type=1) */
if(c->index_hd > 0 && c->idx_type[i] && k3_dsa_indexer_on()){
w_load(m,&a->wk,NM("model.layers.%d.self_attn.w_k.weight",i),(int64_t)c->index_hd,c->hidden,mbits);
w_load(m,&a->wq,NM("model.layers.%d.self_attn.w_q.weight",i),(int64_t)c->index_hd,c->q_lora,mbits);
w_load(m,&a->wp,NM("model.layers.%d.self_attn.w_p.weight",i),(int64_t)c->index_hd,c->hidden,mbits);
a->knw=f32_load(m,NM("model.layers.%d.self_attn.kn_w",i),(int64_t)c->index_hd);
a->knb=f32_load(m,NM("model.layers.%d.self_attn.kn_b",i),(int64_t)c->index_hd);
}
}
if(l->sparse){
Moe *o=&l->moe;
o->router=f32_load(m,NM("model.layers.%d.block_sparse_moe.gate.weight",i),(int64_t)c->n_experts*c->hidden);
o->rbias =f32_load(m,NM("model.layers.%d.block_sparse_moe.gate.e_score_correction_bias",i),c->n_experts);
o->lat_norm=f32_load(m,NM("model.layers.%d.block_sparse_moe.routed_expert_norm.weight",i),c->latent);
w_load(m,&o->lat_down,NM("model.layers.%d.block_sparse_moe.routed_expert_down_proj.weight",i),c->latent,c->hidden,bits);
w_load(m,&o->lat_up,NM("model.layers.%d.block_sparse_moe.routed_expert_up_proj.weight",i),c->hidden,c->latent,bits);
int shi=c->moe_inter*c->n_shared;
w_load(m,&o->sh_gate,NM("model.layers.%d.block_sparse_moe.shared_experts.gate_proj.weight",i),shi,c->hidden,bits);
w_load(m,&o->sh_up,NM("model.layers.%d.block_sparse_moe.shared_experts.up_proj.weight",i),shi,c->hidden,bits);
w_load(m,&o->sh_down,NM("model.layers.%d.block_sparse_moe.shared_experts.down_proj.weight",i),c->hidden,shi,bits);
} else {
w_load(m,&l->d_gate,NM("model.layers.%d.mlp.gate_proj.weight",i),c->dense_inter,c->hidden,bits);
w_load(m,&l->d_up,NM("model.layers.%d.mlp.up_proj.weight",i),c->dense_inter,c->hidden,bits);
w_load(m,&l->d_down,NM("model.layers.%d.mlp.down_proj.weight",i),c->hidden,c->dense_inter,bits);
}
if(i%8==0) fprintf(stderr,"[K3] loaded layer %d/%d (%.1fs, RSS %.1f GB)\n",i+1,c->n_layers,now_s()-t0,rss_gb());
}
snprintf(nm,sizeof(nm),"%smodel.norm.weight",m->pfx);
m->has_head = load_boundaries&&st_has(&m->S,nm);
if(m->has_head){
m->final_norm=f32_load(m,"model.norm.weight",c->hidden);
{ float *rn=f32_load(m,"model.output_attn_res_norm.weight",c->hidden);
float *rp=f32_load(m,"model.output_attn_res_proj.weight",c->hidden);
m->out_sw=falloc(c->hidden); for(int d=0;d<c->hidden;d++) m->out_sw[d]=rn[d]*rp[d];
free(rn); free(rp); }
w_load(m,&m->lm_head,"lm_head.weight",c->vocab,c->hidden,hbits);
} else fprintf(stderr,"[K3] final norm/lm_head not present — trace-only mode\n");
if(g_k3_mmap)
fprintf(stderr,"[K3-MMAP] %llu views, %.1f GB prepared weights/scales file-backed\n",
(unsigned long long)g_k3_mmap_views,g_k3_mmap_bytes/1e9);
#ifdef COLI_VULKAN
/* COLI_VULKAN=1 (K3_VK=1 alike): the device opens once the weights are in RAM,
* a missing one costing a line and nothing else. Whether the routed-expert tier
* will run decides where the dense matrices go by default; the tier starts in
* main (k3_vk_tier_start), after the expert history is read, and the shared
* experts follow it there. A CUDA tier (K3_CUDA=1) wins over this one. */
if(k3_vk_requested()){
k3_vk_aliases();
int any_sparse=0, cuda_on=0;
for(int i=0;i<c->n_layers;i++) any_sparse|=m->L[i].sparse;
#ifdef COLI_CUDA
cuda_on=g_k3_cuda;
#endif
g_k3_vk=coli_vk_init_env_tier("kimi_k3",vkt_wanted()&&any_sparse&&!cuda_on);
k3_dho_decide(m,vkt_wanted()&&any_sparse&&!cuda_on,cuda_on); /* COLI_VK_DENSE_HOST, before the cache is sized */
}
#endif
#ifdef COLI_METAL
{ const char *ev=getenv("K3_METAL");
if(ev&&atoi(ev)&&g_k3_mux_slots>1)
fprintf(stderr,"[K3-METAL] KV_SLOTS=%d: Metal keeps one conversation's KDA state; CPU only\n",g_k3_mux_slots);
else if(ev&&atoi(ev)){
fprintf(stderr,"[K3-METAL] attempting init (K3_METAL=%s)\n", ev);
g_k3_metal=coli_metal_init();
if(!g_k3_metal) fprintf(stderr,"[K3-METAL] FAILED — CPU only\n");
}
if(g_k3_metal)
fprintf(stderr,"[K3-METAL] init OK (fused KDA token GPU: conv_silu×3+l2_norm+kda_state; attn state CPU)\n");
}
#endif
expert_table_init(m);
/* expert LRU cache, per-layer slots from the global budget */
double egb = getenv("K3_EXPERT_GB")?atof(getenv("K3_EXPERT_GB")):8.0;
int nmoe=0; for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) nmoe++;
int cap=(int)((egb*1e9)/((double)m->e_slot*(nmoe?nmoe:1)));
/* floor 1, NOT topk: experts are loaded and consumed one at a time inside
* a token, so slots never need to hold a whole top-k set. A topk floor
* would silently commit topk*nmoe slots (~26 GB on the 93-layer model)
* regardless of K3_EXPERT_GB. */
if(cap<1) cap=1;
if(cap>c->n_experts) cap=c->n_experts;
if(expert_cap_override>0 && cap>expert_cap_override)
cap=expert_cap_override;
/* ---- RAM budget (#855) --------------------------------------------------
* K3_EXPERT_GB used to be the whole story: cap = egb / slot / layers, with
* nothing subtracted for what is already resident, no MemAvailable, no
* reserve for KV or the page cache, and no guard during the run.
*
* Reported on a 256 GB box with K3_EXPERT_GB=220:
*
* 136 slots x 17.5 MB x 93 layers = 216.2 GB cache, once it fills
* + 35.2 GB already resident
* = 251.4 GB peak, of 256
*
* The cache is EMPTY when the init line prints and fills as the session
* runs, so the first request answers and a later one dies -- which is what
* "colibri engine exited unexpectedly" was.
*
* And --ram was inert here. `grep -c RAM_GB kimi_k3.c` returned 0 against 10
* in colibri.c: the one flag a user would reach for to prevent exactly this
* reached the environment and was never read. Same defect shape as #805 and
* #858 -- a mechanism that lands in one engine and not its siblings.
*
* Unlike colibri.c's cap_for_ram this runs AFTER the dense weights are in,
* so the resident set is MEASURED rather than projected. rss_gb() is the
* truth here, not a model of it, which makes this budget the simpler of the
* two rather than the more elaborate. */
{
double resident = rss_gb();
double avail_now = k3_mem_avail();
#ifdef COLI_VULKAN
double resident_host = resident, avail_host = avail_now; /* with the host copies in */
/* the dense weights on the device only: their host copies go once the device has
* them (k3_vk_tier_start); on a discrete GPU that RAM comes back to the machine,
* on one that shares it the device copy takes it */
if(g_k3_dho){
resident -= g_k3_dho_bytes/1e9;
if(!coli_vk_device_shares_ram()) avail_now += g_k3_dho_bytes/1e9;
}
#endif
double ram_env = getenv("RAM_GB") ? atof(getenv("RAM_GB")) : 0.0;
/* Explicit --ram is a ceiling on the WHOLE process. Absent, take 88% of
* what the OS still offers and add what we already hold -- the same
* fraction colibri.c uses, and for the same reason: overshooting means
* an OOM kill mid-generation, which is far worse than a smaller cache. */
double budget = ram_env > 0 ? ram_env : resident + avail_now*0.88;
/* KV is allocated later, at the first request, so it has to be projected
* here. n_layers x max_t x (kv_lora + qk_rope) x 4, skipping KDA layers,
* with the same K3_MAXT default the serve path uses. */
int max_t = context_hint > 0 ? context_hint :
(getenv("K3_MAXT") ? atoi(getenv("K3_MAXT")) : 8192);
if(max_t < 1) max_t = 8192;
int nkv = 0; for(int i=0;i<c->n_layers;i++) if(!m->L[i].kda) nkv++;
double kv_gb = (double)nkv*(double)max_t*(double)(c->kv_lora+c->qk_rope)*4.0/1e9;
/* 2.5 GB page cache -- measured on Linux 2026-07-06: strangling it drops
* buffered pread from ~800 to ~180 MB/s and the last GB of LRU costs
* more in lost bandwidth than it returns. 1.2 GB activations/logits.
* Checkpoints allocate lazily after this plan, so reserve all enabled
* slots now: RAM_GB is a whole-process ceiling even for opt-in memory. */
double ckpt_gb = k3_ckpt_reserve_gb(m,g_k3_ckpt_slots);
double reserve = 2.5 + 1.2 + kv_gb + ckpt_gb;
double for_experts = 0.0;
double slot_gb = (double)m->e_slot/1e9;
int cap_fit = k3_cap_for_ram(budget, resident, reserve, slot_gb,
nmoe, cap, c->n_experts, &for_experts);
#ifdef COLI_VULKAN
if(g_k3_dho){ /* the same plan with the host copies kept, for the line */
double fe_host = 0.0, budget_host = ram_env > 0 ? ram_env : resident_host + avail_host*0.88;
int fit_host = k3_cap_for_ram(budget_host, resident_host, reserve, slot_gb, nmoe, cap, c->n_experts, &fe_host);
int cap_dev = cap_fit < cap ? (cap_fit > 0 ? cap_fit : 1) : cap, cap_host = fit_host < cap ? (fit_host > 0 ? fit_host : 1) : cap;
fprintf(stderr,"[K3] dense weights on the device only: %.4f GB of host copies out of the resident %.4f GB "
"(reserve %.4f GB): %.4f GB for experts instead of %.4f, expert cache %d/layer instead of %d\n",
g_k3_dho_bytes/1e9, resident_host, reserve, for_experts, fe_host, cap_dev, cap_host);
}
#endif
if(cap_fit < cap){
/* Name every term. The user's number is not being ignored, it is
* being clamped, and they cannot check the clamp without the parts. */
fprintf(stderr,"[K3][RAM_GB=%.1f%s] resident %.1f GB + reserve %.1f GB "
"(page cache 2.5, activations 1.2, KV %dx%d %.1f, checkpoints %d %.1f) "
"-> %.1f GB for experts; "
"cache %d->%d/layer (%.1f MB/slot, %d layers; projected peak %.1f GB)\n",
budget, ram_env>0?"":" auto", resident, reserve, nkv, max_t, kv_gb,
g_k3_ckpt_slots,ckpt_gb,
for_experts>0?for_experts:0.0, cap, cap_fit>0?cap_fit:1,
slot_gb*1000.0, nmoe,
resident + reserve + (double)(cap_fit>0?cap_fit:1)*slot_gb*nmoe);
if(getenv("K3_EXPERT_GB"))
fprintf(stderr,"[K3] K3_EXPERT_GB=%.0f does not fit alongside the "
"%.1f GB already resident. It is a request, not a reservation.\n",
egb, resident);
cap = cap_fit;
}
if(cap < 1){
/* Not even one slot per layer. Saying cap=1 and continuing turns
* "does not fit" into "overruns", which is the OOM kill this exists
* to avoid -- and the kernel kills with SIGKILL, so the engine dies
* with no error and no log at all. Refuse, unless told otherwise. */
cap = 1;
double peak = resident + reserve + slot_gb*nmoe;
fprintf(stderr,"[K3] WARNING: cap=1 is the floor and the projected peak is "
"%.1f GB, %.1f GB over the budget.\n", peak, peak-budget);
if(avail_now > 0 && peak > resident + avail_now &&
!(getenv("COLI_RAM_OVERCOMMIT") && atoi(getenv("COLI_RAM_OVERCOMMIT")))){
fprintf(stderr,"[K3] refusing to start: that peak also exceeds the %.1f GB "
"this machine actually has left, so the kernel would kill this run "
"mid-generation.\n[K3] lower K3_MAXT, raise --ram if the box really has "
"it, or set COLI_RAM_OVERCOMMIT=1 to override.\n", resident + avail_now);
exit(2);
}
}
}
{ int ncl=c->n_layers>0?c->n_layers:1;
m->ecache=calloc((size_t)ncl,sizeof(LCache)); }
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse){
m->ecache[i].cap=cap; m->ecache[i].s=calloc(cap,sizeof(Slot));
m->ecache[i].slot_by_expert=malloc((size_t)c->n_experts*sizeof(int));
if(!m->ecache[i].s||!m->ecache[i].slot_by_expert){
fprintf(stderr,"OOM expert cache/index\n"); exit(1); }
for(int j2=0;j2<cap;j2++) m->ecache[i].s[j2].eid=-1;
for(int e=0;e<c->n_experts;e++) m->ecache[i].slot_by_expert[e]=-1;
}
fprintf(stderr,"[K3] init done in %.1fs | %d layers | expert cache %d/layer (%.1f MB/slot) | RSS %.1f GB\n",
now_s()-t0,c->n_layers,cap,m->e_slot/1e6,rss_gb());
if(g_k3_ckpt_slots>0)
fprintf(stderr,"[K3-CKPT] enabled: %d recurrent-state slots (%.1f MB each, %s)\n",
g_k3_ckpt_slots,k3_ckpt_blob_floats(m)*sizeof(float)/1048576.0,
g_k3_ckpt_dir?g_k3_ckpt_dir:"RAM");
/* Validation/capture files are standalone diagnostics. A donor process
* must not inherit them or share their process-global FILE pointers. */
if(!segment_mode){
/* Phase 9: layer validation init */
{ const char *vl=getenv("K3_VALIDATE_LAYER");
const char *vt=getenv("K3_VALIDATE_TOKEN");
if(vl){
g_k3_val_layer=atoi(vl);
g_k3_val_token=vt?atoi(vt):0;
if(g_k3_val_token<0) g_k3_val_token=0;
if(g_k3_val_layer<0||g_k3_val_layer>=c->n_layers){
fprintf(stderr,"[K3-VAL] layer %d out of range (0..%d) — disabled\n",g_k3_val_layer,c->n_layers-1);
g_k3_val_layer=-1;
} else {
const char *ofn=getenv("K3_VALIDATE_OUT");
/* CWD-relative, not a /tmp path: same rule as test_stops.c and
test_pipe_block.c, since the windows job builds native .exe
files that resolve Windows paths and /tmp is not one. The
chosen path is printed on both the success and the failure
branch below, so a caller relying on the old default is told
where the dump went. */
if(!ofn) ofn="k3_val";
g_k3_val_fp=fopen(ofn,"w");
if(!g_k3_val_fp){
fprintf(stderr,"[K3-VAL] cannot open %s (%s) — disabled\n",ofn,strerror(errno));
g_k3_val_layer=-1;
} else {
fprintf(stderr,"[K3-VAL] active: layer %d, token %d, output %s (%d dims)\n",
g_k3_val_layer,g_k3_val_token,ofn,c->hidden);
}
}
}
}
/* K3_DEBUG_OUT: Metal/CPU side-by-side dim comparison */
{ const char *dbg=getenv("K3_DEBUG_OUT");
if(dbg){ g_k3_dfp=fopen(dbg,"w"); if(g_k3_dfp) fprintf(stderr,"[K3-DBG] active: output %s\n",dbg); }
}
/* Phase 10: full model logits capture */
{ const char *vl=getenv("K3_VAL_LOGITS");
if(vl){
g_k3_val_lfp=fopen(vl,"wb");
if(!g_k3_val_lfp){
fprintf(stderr,"[K3-VAL-LOGITS] cannot open %s (%s) — disabled\n",vl,strerror(errno));
} else {
fprintf(stderr,"[K3-VAL-LOGITS] active: output %s (%d dims)\n",vl,c->vocab);
}
}
}
}
#undef NM
}
/* ---------- AttnRes softmax mix over [bres rows..., prefix] ---------- */
static void res_mix(float *out, const float *prefix, const float *bres, int nb, int D,
const float *sw, float eps){
const float *v[16]; float sc[16];
for(int e=0;e<nb;e++) v[e]=bres+(int64_t)e*D;
v[nb]=prefix;
for(int e=0;e<=nb;e++){
double ms=0,dot=0;
for(int d=0;d<D;d++){ double x=v[e][d]; ms+=x*x; dot+=x*(double)sw[d]; }
sc[e]=(float)(dot/sqrt(ms/D+eps));
}
softmax_(sc,nb+1);
for(int d=0;d<D;d++){ float a=0; for(int e=0;e<=nb;e++) a+=sc[e]*v[e][d]; out[d]=a; }
}
/* Phase 9: dump ONE token's vector to output file (.ascii lines: dim0 val0\n dim1 val1\n ...) */
static void val_dump(const char *name, const float *v, int n){
if(!g_k3_val_fp) return;
fprintf(g_k3_val_fp, "%s %d\n",name,n);
for(int i=0;i<n;i++) fprintf(g_k3_val_fp," %d %.16g\n",i,v[i]);
}
/* ---------- KDA layer (chunk of C tokens; projections batched, recurrence
* sequential per token, AVX2 on the state sweeps) ---------- */
static void kda_forward(Model *m, Layer *l, int li, const float *x, int C, float *out){
Cfg *c=&m->c; Kda *a=&l->a;
int P=c->kda_proj, H=c->kda_heads, hd=c->kda_hd, K=c->conv_k;
{ static int once=0; if(li==0 && !once){ once=1;
fprintf(stderr,"[KDA] L%d metal=%d P=%d H=%d hd=%d K=%d\n", li, g_k3_metal, P, H, hd, K); } }
float *q=falloc((int64_t)C*P), *k=falloc((int64_t)C*P), *v=falloc((int64_t)C*P);
float *gp=falloc((int64_t)C*P), *on=falloc((int64_t)C*P);
float *t1=falloc((int64_t)C*c->kda_hd), *graw=falloc((int64_t)C*P), *braw=falloc((int64_t)C*H);
#ifdef COLI_METAL
float *meta_oh = NULL, *meta_alpha = NULL, *meta_beta = NULL;
if(g_k3_metal){
meta_oh = falloc((int64_t)C*P);
meta_alpha = falloc((int64_t)C*P);
meta_beta = falloc((int64_t)C*H);
}
#endif
{ static int once=0; if(li==0 && !once){ once=1;
fprintf(stderr,"[K3-FMT] L%d KDA: q.fmt=%d gs=%d k.fmt=%d gs=%d v.fmt=%d gs=%d g.fmt=%d gs=%d\n",
li, a->q.fmt, a->q.gs, a->k.fmt, a->k.gs, a->v.fmt, a->v.gs, a->g.fmt, a->g.gs);
fflush(stderr); } }
w_matmul(q,x,&a->q,C); w_matmul(k,x,&a->k,C); w_matmul(v,x,&a->v,C);
w_matmul(gp,x,&a->g,C);
if(!K3_F32_DEV(a->vk_fa,t1,x,C,c->hidden,c->kda_hd)){
K3_F32_NEED(a->fa); k3_matmul_f32((void*)&a->metal_fa, t1, x, a->fa, C, c->hidden, c->kda_hd); }
if(!K3_F32_DEV(a->vk_fb,graw,t1,C,c->kda_hd,P)){
K3_F32_NEED(a->fb); k3_matmul_f32((void*)&a->metal_fb, graw, t1, a->fb, C, c->kda_hd, P); }
if(!K3_F32_DEV(a->vk_bp,braw,x,C,c->hidden,H)){
K3_F32_NEED(a->bp); k3_matmul_f32((void*)&a->metal_bp, braw, x, a->bp, C, c->hidden, H); }
float qscale=1.f/sqrtf((float)hd);
for(int t=0;t<C;t++){
float *qt=q+(int64_t)t*P, *kt=k+(int64_t)t*P, *tv=v+(int64_t)t*P;
float *gpt=gp+(int64_t)t*P, *ont=on+(int64_t)t*P;
const float *rgt=graw+(int64_t)t*P, *bt=braw+(int64_t)t*H;
#ifdef COLI_METAL
if(g_k3_metal){
float *mo = meta_oh + (int64_t)t*P;
float *ma = meta_alpha + (int64_t)t*P;
float *mb = meta_beta + (int64_t)t*H;
for(int h=0;h<H;h++){
const float *rgt_h=rgt+(int64_t)h*hd; float *ma_h=ma+(int64_t)h*hd;
for(int i=0;i<hd;i++){
float z=rgt_h[i]+a->dt[(int64_t)h*hd+i];
ma_h[i]=expf(c->gate_lb*sigmoidf_(a->A[h]*z));
}
mb[h]=sigmoidf_(bt[h]);
}
memset(mo, 0, (size_t)P * sizeof(float));
if(!coli_metal_kda_fused_token(m->cwq[li], qt, m->cwk[li], kt, m->cwv[li], tv,
a->conv_q, a->conv_k, a->conv_v, m->kstate[li], ma, mb, mo, P, K, H, hd)){
g_k3_metal = 0;
fprintf(stderr, "[K3-METAL] kda_fused_token L%d t%d failed — falling back to CPU\n", li, t);
} else {
if(g_k3_dfp && li==0 && t==0){
fprintf(stderr,"[META_ENTER] li=%d t=%d g_k3_metal=%d\n",li,t,g_k3_metal);
fprintf(g_k3_dfp,"[META_DIMS] P=%d H=%d hd=%d K=%d kda_proj=%d kda_hd=%d kda_heads=%d\n",
P, H, hd, K, c->kda_proj, c->kda_hd, c->kda_heads);
fprintf(g_k3_dfp,"[META_Q] h0 d0-%d:\n",hd-1);
for(int i=0;i<hd;i++) fprintf(g_k3_dfp," %d %.12g\n",i,qt[i]);
fprintf(g_k3_dfp,"[META_K] h0 d0-%d:\n",hd-1);
for(int i=0;i<hd;i++) fprintf(g_k3_dfp," %d %.12g\n",i,kt[i]);
fprintf(g_k3_dfp,"[META_V] h0 d0-%d:\n",hd-1);
for(int i=0;i<hd;i++) fprintf(g_k3_dfp," %d %.12g\n",i,tv[i]);
fprintf(g_k3_dfp,"[META_OH] h0 d0-%d:\n",hd-1);
for(int i=0;i<hd;i++) fprintf(g_k3_dfp," %d %.12g\n",i,mo[i]);
fprintf(g_k3_dfp,"[META_ALPHA] h0 d0-%d:\n",hd-1);
for(int i=0;i<hd;i++) fprintf(g_k3_dfp," %d %.12g\n",i,ma[i]);
fprintf(g_k3_dfp,"[META_BETA] h0 %.12g\n",mb[0]);
// Compute ratio oh[i]/(tv[i]*mb[0]) for all dims
fprintf(g_k3_dfp,"\n[META_DOT_RATIO] oh[i] / (tv[i] * beta):\n");
for(int i=0;i<hd;i++){
float r = mo[i] / (tv[i] * mb[0]);
fprintf(g_k3_dfp," %d %.12g\n",i,r);
}
fflush(g_k3_dfp);
}
for(int h=0;h<H;h++){
const float *oh_h=mo+(int64_t)h*hd;
double ms=0;
for(int i=0;i<hd;i++) ms+=(double)oh_h[i]*oh_h[i];
float r=1.f/sqrtf((float)(ms/hd)+c->eps);
float *dst=ont+(int64_t)h*hd;
for(int i=0;i<hd;i++)
dst[i]=oh_h[i]*r*a->onw[i]*sigmoidf_(gpt[(int64_t)h*hd+i]);
}
continue;
}
}
#endif
{
const K3Seq *rq=m->mux_rows?m->mux_rows[t].seq:NULL; /* a multiplexed step: the row's own */
float *wins_cpu[3]={rq?rq->cwq[li]:m->cwq[li],rq?rq->cwk[li]:m->cwk[li],rq?rq->cwv[li]:m->cwv[li]};
float *vecs_cpu[3]={qt,kt,tv}; float *taps_cpu[3]={a->conv_q,a->conv_k,a->conv_v};
for(int w2=0;w2<3;w2++){
float *win=wins_cpu[w2], *vec=vecs_cpu[w2]; const float *cw=taps_cpu[w2];
#pragma omp parallel for schedule(static)
for(int d=0;d<P;d++){
float *wd=win+(int64_t)d*K;
for(int j=0;j<K-1;j++) wd[j]=wd[j+1];
wd[K-1]=vec[d];
float acc=0; const float *cd=cw+(int64_t)d*K;
for(int j=0;j<K;j++) acc+=cd[j]*wd[j];
vec[d]=siluf_(acc);
}
}
}
#pragma omp parallel for schedule(static)
for(int h=0;h<H;h++){
const float *qh=qt+(int64_t)h*hd, *kh=kt+(int64_t)h*hd, *vh=tv+(int64_t)h*hd;
float qn[512], kn[512], alpha[512], kS[512], vt[512], oh[512];
{
float sq=0,sk=0;
for(int i=0;i<hd;i++){ sq+=qh[i]*qh[i]; sk+=kh[i]*kh[i]; }
sq=1.f/sqrtf(sq+1e-6f); sk=1.f/sqrtf(sk+1e-6f);
for(int i=0;i<hd;i++){ qn[i]=qh[i]*sq*qscale; kn[i]=kh[i]*sk; }
}
for(int i=0;i<hd;i++){
float z=rgt[(int64_t)h*hd+i]+a->dt[(int64_t)h*hd+i];
alpha[i]=expf(c->gate_lb*sigmoidf_(a->A[h]*z));
}
float beta=sigmoidf_(bt[h]);
float *S=(m->mux_rows?m->mux_rows[t].seq->kstate[li]:m->kstate[li])+(int64_t)h*hd*hd;
memset(kS,0,sizeof(kS));
#ifdef __AVX2__
if(!(hd&7)){
for(int kk=0;kk<hd;kk++){
float *row=S+(int64_t)kk*hd;
__m256 al8=_mm256_set1_ps(alpha[kk]), kv8=_mm256_set1_ps(kn[kk]);
for(int vv=0;vv<hd;vv+=8){
__m256 r=_mm256_mul_ps(_mm256_loadu_ps(row+vv),al8);
_mm256_storeu_ps(row+vv,r);
_mm256_storeu_ps(kS+vv,_mm256_fmadd_ps(kv8,r,_mm256_loadu_ps(kS+vv)));
}
}
for(int vv=0;vv<hd;vv++) vt[vv]=(vh[vv]-kS[vv])*beta;
memset(oh,0,sizeof(oh));
for(int kk=0;kk<hd;kk++){
float *row=S+(int64_t)kk*hd;
__m256 kv8=_mm256_set1_ps(kn[kk]), qq8=_mm256_set1_ps(qn[kk]);
for(int vv=0;vv<hd;vv+=8){
__m256 r=_mm256_fmadd_ps(kv8,_mm256_loadu_ps(vt+vv),_mm256_loadu_ps(row+vv));
_mm256_storeu_ps(row+vv,r);
_mm256_storeu_ps(oh+vv,_mm256_fmadd_ps(qq8,r,_mm256_loadu_ps(oh+vv)));
}
}
} else {
#endif
for(int kk=0;kk<hd;kk++){
float *row=S+(int64_t)kk*hd; float al=alpha[kk], kv=kn[kk];
for(int vv=0;vv<hd;vv++){ row[vv]*=al; kS[vv]+=kv*row[vv]; }
}
for(int vv=0;vv<hd;vv++) vt[vv]=(vh[vv]-kS[vv])*beta;
memset(oh,0,sizeof(oh));
for(int kk=0;kk<hd;kk++){
float *row=S+(int64_t)kk*hd; float kv=kn[kk], qq=qn[kk];
for(int vv=0;vv<hd;vv++){ row[vv]+=kv*vt[vv]; oh[vv]+=qq*row[vv]; }
}
#ifdef __AVX2__
}
#endif
if(g_k3_dfp && li==0 && t==0 && h==0){
fprintf(g_k3_dfp,"[CPU_DIMS] P=%d H=%d hd=%d K=%d\n",P,H,hd,K);
fprintf(g_k3_dfp,"[CPU_QTRAW] h0 d0-%d (conv_silu out):\n",hd-1);
for(int i=0;i<hd;i++) fprintf(g_k3_dfp," %d %.12g\n",i,qh[i]);
fprintf(g_k3_dfp,"[CPU_VTRAW] h0 d0-%d (conv_silu out):\n",hd-1);
for(int i=0;i<hd;i++) fprintf(g_k3_dfp," %d %.12g\n",i,vh[i]);
fprintf(g_k3_dfp,"[CPU_QN] h0 d0-%d (l2_norm q):\n",hd-1);
for(int i=0;i<hd;i++) fprintf(g_k3_dfp," %d %.12g\n",i,qn[i]);
fprintf(g_k3_dfp,"[CPU_KN] h0 d0-%d (l2_norm k):\n",hd-1);
for(int i=0;i<hd;i++) fprintf(g_k3_dfp," %d %.12g\n",i,kn[i]);
fprintf(g_k3_dfp,"[CPU_OH] h0 d0-%d (state out):\n",hd-1);
for(int i=0;i<hd;i++) fprintf(g_k3_dfp," %d %.12g\n",i,oh[i]);
// Compute ratio oh[i]/(vh[i]*beta)
fprintf(g_k3_dfp,"\n[CPU_DOT_RATIO] oh[i] / (vh[i] * beta):\n");
for(int i=0;i<hd;i++){
float r = oh[i] / (vh[i] * beta);
fprintf(g_k3_dfp," %d %.12g\n",i,r);
}
fflush(g_k3_dfp);
}
/* per-head RMSNorm * sigmoid(full-rank gate) */
double ms=0; for(int vv=0;vv<hd;vv++) ms+=(double)oh[vv]*oh[vv];
float r=1.f/sqrtf((float)(ms/hd)+c->eps);
float *dst=ont+(int64_t)h*hd;
for(int vv=0;vv<hd;vv++) dst[vv]=oh[vv]*r*a->onw[vv]*sigmoidf_(gpt[(int64_t)h*hd+vv]);
}
}
w_matmul(out,on,&a->o,C);
free(q);free(k);free(v);free(gp);free(on);free(t1);free(graw);free(braw);
#ifdef COLI_METAL
if(g_k3_metal || meta_oh){
free(meta_oh); free(meta_alpha); free(meta_beta);
}
#endif
}
/* ---------- gated MLA layer (chunk of C tokens, NoPE, absorb; projections
* batched, per-token causal attention — token t attends to 0..pos0+t) ------ */
static void mla_forward(Model *m, Layer *l, int li, const float *x, int pos0, int C, float *out){
Cfg *c=&m->c; Mla *a=&l->m;
int H=c->n_heads, qh=c->qk_head, vh=c->v_head, kvl=c->kv_lora, qr=c->qk_rope;
float *qa=falloc((int64_t)C*c->q_lora), *qv=falloc((int64_t)C*H*qh);
float *ckv=falloc((int64_t)C*(kvl+qr));
float *gv=falloc((int64_t)C*H*vh), *ctx=falloc((int64_t)C*H*vh);
{ static int once=0; if(!once){ once=1;
fprintf(stderr,"[K3-FMT] L%d MLA: qa.fmt=%d gs=%d qb.fmt=%d gs=%d kva.fmt=%d gs=%d\n",
li, a->qa.fmt, a->qa.gs, a->qb.fmt, a->qb.gs, a->kva.fmt, a->kva.gs);
fflush(stderr); } }
w_matmul(qa,x,&a->qa,C);
for(int t=0;t<C;t++)
rmsnorm_(qa+(int64_t)t*c->q_lora,qa+(int64_t)t*c->q_lora,a->qa_ln,c->q_lora,c->eps);
w_matmul(qv,qa,&a->qb,C);
w_matmul(ckv,x,&a->kva,C);
/* KV cache write stays on CPU even under Metal. The MLA attention loop below reads the
* cache directly from host memory (m->Lc[li]/m->Rc[li]), so the write must land there.
* coli_metal_kv_write() wrote into an unaligned wrap() copy that was never synced back,
* and at row 0 instead of row pos_base — so on Metal the host cache was never populated
* and attention read garbage, collapsing decode after a few tokens. This write is a
* trivial rmsnorm+copy per row (negligible vs MoE), so CPU is the correct home for it. */
const K3Row *mr = m->mux_rows; /* a multiplexed step: each row's own caches and position */
for (int t = 0; t < C; t++) {
int pt = mr ? mr[t].pos : pos0 + t;
float *Lrow = (mr ? mr[t].seq->Lc[li] : m->Lc[li]) + (int64_t)pt * kvl,
*Rrow = (mr ? mr[t].seq->Rc[li] : m->Rc[li]) + (int64_t)pt * qr;
const float *cv = ckv + (int64_t)t * (kvl + qr);
rmsnorm_(Lrow, cv, a->kva_ln, kvl, c->eps);
memcpy(Rrow, cv + kvl, qr * sizeof(float));
}
/* DSA indexer: prefill — write K portion for full layers */
if(c->index_hd > 0 && c->idx_type[li] && k3_dsa_indexer_on()){
/* the chunk's keys in one call ([C, index_hd] from [C, hidden], rows contiguous
* in Ic; every kernel computes a row as the one-row call does), then per row */
float *ikb = mr ? falloc((int64_t)C * c->index_hd) : a->Ic + (int64_t)pos0 * c->index_hd;
w_matmul(ikb, x, &a->wk, C);
for(int t=0;t<C;t++){
int pt = mr ? mr[t].pos : pos0 + t;
float *ikd = mr ? mr[t].seq->Ic[li] + (int64_t)pt * c->index_hd : a->Ic + (int64_t)pt * c->index_hd;
if (mr) memcpy(ikd, ikb + (int64_t)t * c->index_hd, (size_t)c->index_hd * sizeof(float));
rmsnorm_(ikd, ikd, a->knw, c->index_hd, c->eps);
if(c->qk_rope > 0)
dsa_rope(ikd, pt, c->qk_rope, c->theta); /* in-place on first qk_rope dims */
}
if (mr) free(ikb);
}
w_matmul(gv,x,&a->g,C);
#ifdef COLI_VULKAN
if(a->kvb.vk_gone) k3_w_reload(&a->kvb); /* its rows are read on the CPU below */
#endif
for(int tt=0;tt<C;tt++){
int nt=(mr?mr[tt].pos:pos0+tt)+1;
const float *Lc_=mr?mr[tt].seq->Lc[li]:m->Lc[li], *Rc_=mr?mr[tt].seq->Rc[li]:m->Rc[li];
const float *qvt=qv+(int64_t)tt*H*qh, *gvt=gv+(int64_t)tt*H*vh;
float *ctxt=ctx+(int64_t)tt*H*vh;
#pragma omp parallel for schedule(static)
for(int h=0;h<H;h++){
const float *qp=qvt+(int64_t)h*qh, *qrp=qp+c->qk_nope;
int rbase=h*(c->qk_nope+vh);
float qabs[4096]; memset(qabs,0,kvl*sizeof(float));
for(int d=0;d<c->qk_nope;d++) w_addrow(&a->kvb,rbase+d,qp[d],qabs);
float *sc=falloc(nt);
for(int t=0;t<nt;t++){
const float *Lt=Lc_+(int64_t)t*kvl, *Rt=Rc_+(int64_t)t*qr;
float s2=0; for(int i=0;i<kvl;i++) s2+=qabs[i]*Lt[i];
for(int i=0;i<qr;i++) s2+=qrp[i]*Rt[i];
sc[t]=s2*c->attn_scale;
}
softmax_(sc,nt);
float clat[4096]; memset(clat,0,kvl*sizeof(float));
for(int t=0;t<nt;t++){
const float *Lt=Lc_+(int64_t)t*kvl; float s2=sc[t];
for(int i=0;i<kvl;i++) clat[i]+=s2*Lt[i];
}
free(sc);
float *cx=ctxt+(int64_t)h*vh;
for(int d=0;d<vh;d++)
cx[d]=w_rowdot(&a->kvb,rbase+c->qk_nope+d,clat)*sigmoidf_(gvt[(int64_t)h*vh+d]);
}
}
w_matmul(out,ctx,&a->o,C);
free(qa);free(qv);free(ckv);free(gv);free(ctx);
}
/* ---------- routed experts: LRU + pread from the shards ----------
* Loads are issued in PARALLEL (OMP over the token's misses, working-set
* slots) and, when K3_DIRECT=1 (default) and st.h has an O_DIRECT twin fd,
* bypass the page cache: measured on the box 7.1 GB/s direct vs 2.9 buffered
* (and ~1.8 effective once the resident weights leave no cache headroom). */
/* Cache slots are serially published by cache_promote() or pin_seed(). Expert
* reads land in ws[] first and never enter this index until their bytes are
* complete. Validate both the local range and identity before returning: a
* broken invariant must become a miss, never another expert's weights. */
#ifdef COLI_CACHE_INDEX_TEST
static uint64_t g_slot_index_probes;
#define SLOT_INDEX_PROBE() (g_slot_index_probes++)
#else
#define SLOT_INDEX_PROBE() ((void)0)
#endif
/* One byte per expert: routed in this turn or not. The dashboard's Brain tab
* reads it as the HITS bitmap after every turn (serve_hits), cleared there. */
static pthread_mutex_t g_ehit_mx = PTHREAD_MUTEX_INITIALIZER;
static void ehit_mark(Model *m, int li, int eid){
Cfg *c=&m->c;
/* The first touch can come from a parallel region (qwen36: the tier
* warmstart's omp loop calls expert_get from twelve threads at once):
* one thread published m->ehit while it was still filling the rows and
* a sibling dereferenced m->ehit[layer] == NULL -- SIGSEGV in about one
* run in twelve on a CUDA warmstart. Build the table privately, publish
* it once under a lock (double-checked), and read it with acquire order. */
uint8_t **ehit=__atomic_load_n(&m->ehit,__ATOMIC_ACQUIRE);
if(!ehit){
pthread_mutex_lock(&g_ehit_mx);
ehit=m->ehit;
if(!ehit){
ehit=calloc((size_t)c->n_layers,sizeof(uint8_t*));
for(int i=0;i<c->n_layers;i++) ehit[i]=calloc((size_t)c->n_experts,1);
__atomic_store_n(&m->ehit,ehit,__ATOMIC_RELEASE);
}
pthread_mutex_unlock(&g_ehit_mx);
}
if(li>=0&&li<c->n_layers&&eid>=0&&eid<c->n_experts) ehit[li][eid]=1;
}
static Slot *slot_indexed(Model *m, int li, int eid){
LCache *lc=&m->ecache[li];
if(eid<0||eid>=m->c.n_experts||!lc->slot_by_expert) return NULL;
SLOT_INDEX_PROBE();
int i=lc->slot_by_expert[eid];
if(i<0||i>=lc->n||lc->s[i].eid!=eid) return NULL;
return &lc->s[i];
}
static Slot *slot_find(Model *m, int li, int eid){
Slot *s=slot_indexed(m,li,eid);
if(s){ m->hits++; s->used=++m->clock; }
return s;
}
static void cache_unindex(Model *m,int li,Slot *s){
LCache *lc=&m->ecache[li]; int i=(int)(s-lc->s);
if(s->eid>=0&&s->eid<m->c.n_experts&&lc->slot_by_expert[s->eid]==i)
lc->slot_by_expert[s->eid]=-1;
}
static void cache_index(Model *m,int li,Slot *s){
LCache *lc=&m->ecache[li]; int i=(int)(s-lc->s);
if(s->eid<0||s->eid>=m->c.n_experts||i<0||i>=lc->n){
fprintf(stderr,"layer %d: invalid expert cache publication\n",li); exit(1); }
lc->slot_by_expert[s->eid]=i;
}
static void cache_promote(Model *m,int li,Slot *dst,Slot *loaded){
cache_unindex(m,li,dst);
Slot tmp=*dst; *dst=*loaded; *loaded=tmp;
dst->used=++m->clock;
cache_index(m,li,dst);
}
static void expert_read(Model *m, int li, int eid, Slot *s){
if(!s->base){
if(posix_memalign((void**)&s->base,4096,(size_t)m->e_slot+8192)){
fprintf(stderr,"OOM expert slot\n"); exit(1); }
}
ERef *er=&m->eref[(int64_t)li*m->c.n_experts+eid];
int64_t sizes[6]={m->e_w1p,m->e_w1s,m->e_w2p,m->e_w2s,m->e_w1p,m->e_w1s};
if(er->fd[0]<0){ fprintf(stderr,"[K3] expert L%d E%d missing on disk\n",li,eid); exit(1); }
if(er->contig){
int dfd = g_k3_direct ? st_direct_fd(&m->S,er->fd[0]) : -1;
if(dfd>=0){
/* aligned window read; sub-4K head/tail slack handled explicitly.
* The tail past the last aligned block (or past EOF) is fetched
* with a tiny buffered pread — O_DIRECT wants aligned lengths. */
int64_t a0=er->off[0]&~4095LL, pad=er->off[0]-a0;
int64_t want=pad+m->e_slot;
struct stat sb;
int64_t dlen=(want+4095)&~4095LL;
if(fstat(dfd,&sb)==0 && a0+dlen>sb.st_size) dlen=(sb.st_size-a0)&~4095LL;
if(dlen>0) st_pread_full(dfd,s->base,dlen,a0,"pread expert direct");
if(dlen<want)
st_pread_full(er->fd[0],s->base+dlen,want-dlen,a0+dlen,"pread expert tail");
s->buf=s->base+pad;
} else {
st_pread_full(er->fd[0],s->base,m->e_slot,er->off[0],"pread expert");
s->buf=s->base;
}
} else {
uint8_t *dst=s->base;
for(int k=0;k<6;k++){
if(er->fd[k]<0){ fprintf(stderr,"[K3] expert L%d E%d tensor %d missing on disk\n",li,eid,k); exit(1); }
st_pread_full(er->fd[k],dst,sizes[k],er->off[k],"pread expert");
dst+=sizes[k];
}
s->buf=s->base;
}
s->eid=eid;
}
static inline float situf_(float g, float u, float b1, float b2){
return b1*tanhf(g/b1)*sigmoidf_(g) * b2*tanhf(u/b2);
}
#ifdef COLI_CUDA
/* Keep SiTU-GLU and intermediate activations on the GPU when supported.
* Older DLLs lack the optional fused entry point, so retain the three-matmul
* path as fallback. Accumulate into u only after a complete expert succeeds. */
static int cuda_expert_apply(Model *m, const uint8_t *w1p, const uint8_t *w1s,
const uint8_t *w2p, const uint8_t *w2s,
const uint8_t *w3p, const uint8_t *w3s,
const float *z, float wk,
float *u, float *gate, float *up, float *hz){
Cfg *c=&m->c;
if(!coli_cuda_expert_mxfp4(hz,z,w1p,w1s,w3p,w3s,w2p,w2s,
1,c->latent,c->moe_inter,c->situ_b1,c->situ_b2)) {
if(!coli_cuda_matmul_mxfp4(gate,z,w1p,w1s,1,c->latent,c->moe_inter)) return 0;
if(!coli_cuda_matmul_mxfp4(up, z,w3p,w3s,1,c->latent,c->moe_inter)) return 0;
for(int i=0;i<c->moe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);
if(!coli_cuda_matmul_mxfp4(hz,gate,w2p,w2s,1,c->moe_inter,c->latent)) return 0;
}
for(int i=0;i<c->latent;i++) u[i]+=wk*hz[i];
return 1;
}
#endif
/* u += wk * E(z) for one loaded expert slot (SiTU-GLU in the latent).
* gate/up are [moe_inter] scratch, hz is [latent] scratch. u == NULL (the
* Vulkan tier's steps): E(z) is left in hz, for the caller to add in order.
* zq/zsc: riga di z già quantizzata a livello layer da mxfp4_qx (NULL = come
* prima, quantizza in-chiamata). Gate e up la condividono; il down riquantizza
* comunque il proprio input per-expert (gate), oggi però in scratch, non malloc.
* EN: zq/zsc = layer-hoisted pre-quantized z row (NULL = quantize in-call as
* before). Gate and up share it; down still quantizes its per-expert input. */
static void expert_apply(Model *m, Slot *s, const float *z, float wk,
float *u, float *gate, float *up, float *hz,
const int8_t *zq, const float *zsc){
Cfg *c=&m->c;
uint8_t *w1p=s->buf, *w1s=w1p+m->e_w1p, *w2p=w1s+m->e_w1s, *w2s=w2p+m->e_w2p,
*w3p=w2s+m->e_w2s, *w3s=w3p+m->e_w1p;
#ifdef COLI_CUDA
/* Opt-in (K3_CUDA=1), decode only: at S>1 the CPU kernels amortise across
* the batch and the per-call upload would not pay for itself. */
if(g_k3_cuda && u && cuda_expert_apply(m,w1p,w1s,w2p,w2s,w3p,w3s,z,wk,u,gate,up,hz)) return;
#endif
void (*mm)(float*,const float*,const uint8_t*,const uint8_t*,int,int,int)
= g_k3_idot ? matmul_mxfp4_i8 : matmul_mxfp4;
if(g_k3_idot && zq){
matmul_mxfp4_i8_pre(gate,zq,zsc,w1p,w1s,1,c->latent,c->moe_inter);
matmul_mxfp4_i8_pre(up, zq,zsc,w3p,w3s,1,c->latent,c->moe_inter);
} else {
mm(gate,z,w1p,w1s,1,c->latent,c->moe_inter);
mm(up,z,w3p,w3s,1,c->latent,c->moe_inter);
}
for(int i=0;i<c->moe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);
mm(hz,gate,w2p,w2s,1,c->moe_inter,c->latent);
if(u) for(int i=0;i<c->latent;i++) u[i]+=wk*hz[i];
}
#ifdef COLI_VULKAN
/* An expert's bytes as the shared Vulkan tier reads them: gate w1, up w3, down
* w2, each e2m1 pairs with one ue8m0 byte per 32 columns, as the shards hold
* them (b = Slot.buf: w1, w1 scales, w2, w2 scales, w3, w3 scales). */
static VktExpertSrc k3_vk_src(const Model *m, const uint8_t *b){
const uint8_t *w1p=b, *w1s=w1p+m->e_w1p, *w2p=w1s+m->e_w1s, *w2s=w2p+m->e_w2p,
*w3p=w2s+m->e_w2s, *w3s=w3p+m->e_w1p;
return (VktExpertSrc){w1p,w3p,w2p,w1s,w3s,w2s};
}
#endif
/* ---------- async loader pool (K3_PIPE): expert preads overlap compute ----
* A batch of jobs is submitted per token+layer; the compute loop below waits
* per-expert on its ready flag, so expert j's math runs while j+1.. load.
* One batch in flight at a time (the submitter consumes every job before the
* next submit), so the flags need no generation counter. */
#define LP_MAX 64
typedef struct { int li, eid; Slot *s; } LJob;
static struct {
pthread_t th[16]; int nth, started;
pthread_mutex_t mx; pthread_cond_t cv;
Model *m;
LJob job[LP_MAX];
_Atomic int ready[LP_MAX];
_Atomic int next; int count;
} g_lp = { .mx=PTHREAD_MUTEX_INITIALIZER, .cv=PTHREAD_COND_INITIALIZER };
static void *lp_main(void *arg){
(void)arg;
for(;;){
pthread_mutex_lock(&g_lp.mx);
while(atomic_load_explicit(&g_lp.next,memory_order_relaxed)>=g_lp.count)
pthread_cond_wait(&g_lp.cv,&g_lp.mx);
int idx=atomic_fetch_add_explicit(&g_lp.next,1,memory_order_relaxed);
pthread_mutex_unlock(&g_lp.mx);
if(idx>=g_lp.count) continue;
LJob *j=&g_lp.job[idx];
expert_read(g_lp.m,j->li,j->eid,j->s);
atomic_store_explicit(&g_lp.ready[idx],1,memory_order_release);
}
return NULL;
}
static void lp_start(void){
if(g_lp.started) return;
g_lp.nth = getenv("K3_LOAD_THREADS")?atoi(getenv("K3_LOAD_THREADS")):4;
if(g_lp.nth<1) g_lp.nth=1;
if(g_lp.nth>16) g_lp.nth=16;
g_lp.count=0; atomic_store(&g_lp.next,0);
for(int i=0;i<g_lp.nth;i++)
if(pthread_create(&g_lp.th[i],NULL,lp_main,NULL)){
fprintf(stderr,"[K3] K3_PIPE: pthread_create failed, falling back\n");
g_k3_pipe=0; g_lp.nth=i; break;
}
g_lp.started=1;
}
static void lp_submit(Model *m, int n){
pthread_mutex_lock(&g_lp.mx);
g_lp.m=m;
for(int q=0;q<n;q++) atomic_store_explicit(&g_lp.ready[q],0,memory_order_relaxed);
atomic_store_explicit(&g_lp.next,0,memory_order_relaxed);
g_lp.count=n;
pthread_cond_broadcast(&g_lp.cv);
pthread_mutex_unlock(&g_lp.mx);
}
/* the general expert pass for one layer over a CHUNK of positions: nu unique
* experts, expert j applied to pcnt[j] positions (poslist/wlist rows starting
* at pfirst[j]). Loads run in blocks of <=LP_MAX working-set slots, pipelined
* with compute under K3_PIPE (expert j's matmuls overlap expert j+1's read),
* else all-parallel up front. Z/U are [C, stride] position-major.
* R != NULL (the Vulkan tier's steps): nothing is added to U; the output of
* list entry i is written to row rslot[i] of R instead, unweighted, so the
* caller can add the CPU's and the device's experts in one order. */
static void experts_apply_union(Model *m, int li, int nu, const int *uids,
const int *pfirst, const int *pcnt,
const int *poslist, const float *wlist,
const float *Z, int stride, float *U,
float *gate, float *up, float *hz,
float *R, const int *rslot){
/* Quantizzazione sollevata a livello layer (specchia colibri.c #1071):
* ogni riga di Z veniva riquantizzata 2x per OGNI expert che la consuma
* (gate+up dentro matmul_mxfp4_i8, con un malloc/free a chiamata) — con
* topk=8 sono 16 quantizzazioni della stessa riga per layer. Una sola
* passata mxfp4_qx qui produce le stesse righe int8 (ordine identico:
* bit-identico), e il fallimento di allocazione degrada al vecchio path.
* EN: layer-hoisted activation quantization, mirroring colibri.c (#1071).
* One mxfp4_qx pass replaces 2 requantizations per (expert, position);
* allocation failure degrades to the old in-call path. */
int8_t *zq_all=NULL; float *zsc_all=NULL; int zng=0;
if(g_k3_idot && stride%32==0 && nu>0){
int maxt=-1;
for(int j=0;j<nu;j++) for(int p2=0;p2<pcnt[j];p2++)
if(poslist[pfirst[j]+p2]>maxt) maxt=poslist[pfirst[j]+p2];
if(maxt>=0){
zng=stride/32;
zq_all=malloc((size_t)(maxt+1)*stride);
zsc_all=malloc((size_t)(maxt+1)*zng*sizeof(float));
if(zq_all&&zsc_all) mxfp4_qx(Z,maxt+1,stride,zq_all,zsc_all);
else { free(zq_all); free(zsc_all); zq_all=NULL; zsc_all=NULL; }
}
}
for(int base=0;base<nu;base+=LP_MAX){
int nb=nu-base<LP_MAX?nu-base:LP_MAX;
Slot *use[LP_MAX]; int missk[LP_MAX]; int qof[LP_MAX]; int nmiss=0;
for(int j=0;j<nb;j++){
ehit_mark(m,li,uids[base+j]);
use[j]=slot_find(m,li,uids[base+j]); qof[j]=-1;
if(!use[j]){ m->miss++; use[j]=&m->ws[nmiss]; qof[j]=nmiss; missk[nmiss++]=j; }
}
if(nmiss){
if(g_k3_pipe && !g_lp.started) lp_start();
if(g_k3_pipe){
for(int q=0;q<nmiss;q++){
g_lp.job[q].li=li; g_lp.job[q].eid=uids[base+missk[q]]; g_lp.job[q].s=&m->ws[q];
}
lp_submit(m,nmiss);
} else {
double t0=now_s();
#pragma omp parallel for schedule(dynamic,1)
for(int q=0;q<nmiss;q++) expert_read(m,li,uids[base+missk[q]],&m->ws[q]);
m->t_eload+=now_s()-t0;
}
m->ebytes+=(uint64_t)nmiss*(uint64_t)m->e_slot;
}
for(int j=0;j<nb;j++){
if(g_k3_pipe && qof[j]>=0 &&
!atomic_load_explicit(&g_lp.ready[qof[j]],memory_order_acquire)){
double t0=now_s(); /* t_eload = UN-hidden I/O (wait) time */
while(!atomic_load_explicit(&g_lp.ready[qof[j]],memory_order_acquire))
usleep(50);
m->t_eload+=now_s()-t0;
}
#ifdef COLI_VULKAN
/* #848: offer EVERY expert used this layer, not only the ones that just
* came off disk. qof[j]>=0 means "this was a RAM-cache miss", which is the
* right gate for the readiness wait above and the wrong one here: it made
* the VRAM tier reachable only through disk reads, so the fill stopped the
* moment the RAM cache went warm and never resumed. Re-offering a resident
* or queued expert is nearly free (vkt_note returns on its state first),
* and it reads only s->buf, which an LRU slot and a ws[] slot populate
* identically; the tier copies the bytes before this slot can move.
* Rows mode runs while a device batch is in flight: k3_vk_step offers
* those experts after the join instead. */
if(!R&&vkt_ready()){ VktExpertSrc vs=k3_vk_src(m,use[j]->buf); vkt_note(li,uids[base+j],&vs); }
#endif
int f=pfirst[base+j];
for(int p2=0;p2<pcnt[base+j];p2++){
int t=poslist[f+p2];
expert_apply(m,use[j],Z+(int64_t)t*stride,wlist[f+p2],
R?NULL:U+(int64_t)t*stride,gate,up,
R?R+(int64_t)rslot[f+p2]*stride:hz,
zq_all?zq_all+(int64_t)t*stride:NULL,
zq_all?zsc_all+(int64_t)t*zng:NULL);
}
}
/* promotion: swap the freshly-read slots into the layer LRU */
LCache *lc=&m->ecache[li];
int promo = nmiss<lc->cap ? nmiss : lc->cap;
for(int a=0;a<promo;a++){
int q=nmiss-1-a; Slot *dst;
if(lc->n<lc->cap) dst=&lc->s[lc->n++];
else {
/* LRU over the UNPINNED slots. pin_seed caps pinned at cap/2, so a
* victim always exists; the -1 fallback is a belt-and-braces guard
* against a future caller pinning everything and deadlocking here. */
int lru=-1, dev=-1; /* an expert the Vulkan tier holds goes first (vkt_ram_first) */
for(int i=0;i<lc->n;i++){
if(lc->s[i].pinned) continue;
if(lc->s[i].eid>=0 && vkt_ram_first(li,lc->s[i].eid)){ if(dev<0 || lc->s[i].used<lc->s[dev].used) dev=i; continue; }
if(lru<0 || lc->s[i].used<lc->s[lru].used) lru=i;
}
if(dev>=0){ lru=dev; vkt_ram_gave(); }
if(lru<0) break; /* every slot pinned: keep the read */
dst=&lc->s[lru];
}
cache_promote(m,li,dst,&m->ws[q]);
}
}
free(zq_all); free(zsc_all);
}
/* ---- AUTOPIN: seed the layer caches from the accumulated history ----------
*
* K3 counted every routed selection (rt_route) and could READ a history file,
* but only when COLI_USAGE named one by hand, and it never WROTE one — so the
* read path was unreachable for anyone who did not already have a profile from
* somewhere else. colibri.c has seeded itself from <snap>/.coli_usage since the
* beginning and inkling.c does too; this brings K3 in line.
*
* The quota scales with CONFIDENCE in the history, exactly as colibri.c does it:
* a handful of turns is a bad predictor and pinning on it steals slots from the
* LRU, which at least adapts to the session in front of it. Below 5,000 recorded
* selections nothing is pinned at all; the quota reaches its cap of half the
* layer budget at 200,000, which is a few hours of real use.
*
* Cost is bounded and paid once: at most cap/2 experts per sparse layer, read
* sequentially at startup rather than demand-faulted mid-token. */
static void k3_usage(const char *argv0){
fprintf(stderr,
"usage: %s <model_dir> [prompt] [options]\n"
"\n"
" --chat apply the K3 chat template to the prompt\n"
" --system TEXT system message (implies --chat)\n"
" --ngen N tokens to generate (default 32)\n"
" --ids \"1 2 3\" raw token ids instead of a prompt\n"
" (needed when the snapshot has no tokenizer.json;\n"
" generate one with tools/k3_tokenizer.py)\n"
"\n"
" %s /path/to/kimi \"What is the capital of France?\" --ngen 64\n"
"\n"
"For an interactive session use the launcher instead, which starts this\n"
"engine for you and does not need the GLM engine built:\n"
" coli chat --model /path/to/kimi\n"
"\n"
"environment: SERVE=1 serve mode (SNAP=<dir>) - COLI_VULKAN=1 GPU path\n"
" OMP_NUM_THREADS=<physical cores> - AUTOPIN=0 no pin seeding\n",
argv0, argv0);
}
static void pin_seed(Model *m, int64_t hist){
Cfg *c=&m->c;
if(getenv("AUTOPIN") && atoi(getenv("AUTOPIN"))==0) return;
if(hist<5000) return;
double conf=(double)hist/200000.0; if(conf>1) conf=1;
int pinned_total=0;
for(int li=0; li<c->n_layers; li++){
if(!m->L[li].sparse) continue;
LCache *lc=&m->ecache[li];
int quota=(int)(lc->cap*0.5*conf);
if(quota<1) continue;
if(quota>lc->cap/2) quota=lc->cap/2;
const uint32_t *u=rt_counts(li);
if(!u) continue;
/* top-`quota` by recorded use: a partial selection sort, since quota is
* small (half a cache) and E is 896 — no allocation, no qsort callback. */
for(int slot=0; slot<quota && lc->n<lc->cap; slot++){
int best=-1; uint32_t bestc=0;
for(int e=0;e<c->n_experts;e++){
if(u[e]<=bestc) continue;
int taken=slot_indexed(m,li,e)!=NULL;
if(!taken){ best=e; bestc=u[e]; }
}
if(best<0) break; /* history is exhausted */
Slot *d=&lc->s[lc->n];
expert_read(m,li,best,d);
d->eid=best; d->used=++m->clock; d->pinned=1;
lc->n++; cache_index(m,li,d); pinned_total++;
}
}
if(pinned_total)
fprintf(stderr,"[PIN] %d experts pinned from history (%lld selections, %.0f%% confidence)\n",
pinned_total,(long long)hist,100.0*conf);
}
#ifdef COLI_VULKAN
/* ---- the device's place in a run (COLI_VULKAN=1) ----------------------------
* After the history is read: the routed-expert tier (vk_tier.c) with a warm start
* from that history, then the shared experts where coli_vk_dense() puts the dense
* matrices. Its reports: one "[VK] kimi_k3" line of device matmuls and the tier's
* line, at the end of a run and of every serve turn. */
#define K3_VK_ROWS 64 /* positions whose pairs one device batch carries at most */
/* the dense chain (kimi_k3_chain.h, included before step_chunk_ex) */
static void k3c_start(Model *m, int tier_on);
static void k3c_atexit(void);
static void k3c_adopt_shared(Model *m);
static void k3c_report(void);
static int g_k3c_on;
static int k3_vk_in_ram(void *ctx, int li, int e){ return slot_indexed((Model*)ctx,li,e)!=NULL; }
static int k3_vk_load(void *ctx,int li,int e,VktExpertSrc *src,void **h);
static int k3_vk_load_batch(void *ctx,int li,const int *e,int n,VktExpertSrc *srcs,void **h);
static void k3_vk_unhold(void *ctx,void *h);
static size_t k3_w_bytes(const W *w){
int fmt=k3_vk_fmt(w);
if(fmt==10) return (size_t)w->O*w->I*4;
if(fmt==1) return (size_t)w->O*w->I+(size_t)w->O*4;
if(fmt==4) return (size_t)w->O*((w->I+1)/2)+(size_t)w->O*((w->I+w->gs-1)/w->gs)*4;
return 0;
}
/* the shared experts: the dense matrices this engine puts on the device */
static size_t k3_vk_dense_bytes(const Model *m){
size_t b=0;
for(int i=0;i<m->c.n_layers;i++){
if(!m->L[i].sparse) continue;
const Moe *o=&m->L[i].moe;
b+=k3_w_bytes(&o->sh_gate)+k3_w_bytes(&o->sh_up)+k3_w_bytes(&o->sh_down);
}
return b;
}
static int k3_expert_on_disk(const Model *m, int li, int e){
const ERef *er=&m->eref[(int64_t)li*m->c.n_experts+e];
if(er->contig) return er->fd[0]>=0;
for(int k=0;k<6;k++) if(er->fd[k]<0) return 0;
return 1;
}
static void k3_dho_upload(W *w, size_t *b, int *n){ (void)b; if(k3_w_host_bytes(w)&&!w->mapped&&w_vk_upload(w)) (*n)++; }
static void k3_dho_drop(W *w, size_t *b, int *n){
if(!w->vk||w->vk_gone||!w->vk_name||w->mapped) return;
size_t x=k3_w_host_bytes(w);
if(!x) return;
free(w->f); free(w->q8); free(w->q4); free(w->s);
w->f=NULL; w->q8=NULL; w->q4=NULL; w->s=NULL; w->vk_gone=1;
coli_vk_dense_host_dropped(x);
*b+=x; (*n)++;
}
static ColiVkTensor *k3c_f32_tensor(int li, int which); /* kimi_k3_chain.h */
static void k3_dho_f32_drop(Model *m, float **p, const char *name, int64_t n){
if(!*p) return;
g_k3_f32=realloc(g_k3_f32,(size_t)(g_k3_f32_n+1)*sizeof(*g_k3_f32));
if(!g_k3_f32){fprintf(stderr,"OOM f32 home\n");exit(1);}
K3F32Home *h=&g_k3_f32[g_k3_f32_n++];
h->p=p; h->n=n; snprintf(h->name,sizeof h->name,"%s",name);
free(*p); *p=NULL;
coli_vk_dense_host_dropped((size_t)n*4);
(void)m;
}
/* The W matrices up first, into their own vk, so the chain adopts the same copies; the
* host copies go once the chain is set up, and the f32 ones the chain holds with them. */
static void k3_dho_place(Model *m){
size_t b=0; int n=0;
k3_dho_each(m,k3_dho_drop,&b,&n);
if(g_k3c_on){
const Cfg *c=&m->c; char nm[200];
for(int i=0;i<c->n_layers&&i<g_k3_dho_layers;i++){ /* the chain's layers: it holds their f32 matrices */
Layer *l=&m->L[i];
if(l->sparse){
snprintf(nm,sizeof nm,"model.layers.%d.block_sparse_moe.gate.weight",i);
l->moe.vk_router=k3c_f32_tensor(i,0);
k3_dho_f32_drop(m,&l->moe.router,nm,(int64_t)c->n_experts*c->hidden);
}
if(l->kda){
l->a.vk_fa=k3c_f32_tensor(i,1); l->a.vk_fb=k3c_f32_tensor(i,2); l->a.vk_bp=k3c_f32_tensor(i,3);
snprintf(nm,sizeof nm,"model.layers.%d.self_attn.f_a_proj.weight",i);
k3_dho_f32_drop(m,&l->a.fa,nm,(int64_t)c->kda_hd*c->hidden);
snprintf(nm,sizeof nm,"model.layers.%d.self_attn.f_b_proj.weight",i);
k3_dho_f32_drop(m,&l->a.fb,nm,(int64_t)c->kda_proj*c->kda_hd);
snprintf(nm,sizeof nm,"model.layers.%d.self_attn.b_proj.weight",i);
k3_dho_f32_drop(m,&l->a.bp,nm,(int64_t)c->kda_heads*c->hidden);
}
}
}
coli_vk_dense_host_placed("kimi_k3",g_k3c_on
? "the embedding (read from disk per token), norms, vectors and the AttnRes weights"
: "the embedding (read from disk per token), norms, vectors, the AttnRes weights, the routers and the KDA decay and beta projections (f32: without the chain the CPU computes them)");
}
static void k3_vk_tier_start(Model *m){
Cfg *c=&m->c;
if(!g_k3_vk) return;
int nmoe=0, cap=0, cuda_on=0;
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse){ nmoe++; if(m->ecache[i].cap>cap) cap=m->ecache[i].cap; }
#ifdef COLI_CUDA
cuda_on=g_k3_cuda;
#endif
/* COLI_VK_DENSE_HOST: every W into its own vk first (with the chain's fit, its setup
* puts each of its layers' there, layer by layer) */
if(g_k3_dho&&g_k3_fit.L<1){ size_t b=0; int n=0; k3_dho_each(m,k3_dho_upload,&b,&n); }
k3c_start(m,nmoe&&vkt_wanted()&&!cuda_on); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */
if(g_k3_dho&&g_k3_fit.L>0&&!g_k3c_on){ size_t b=0; int n=0; k3_dho_each(m,k3_dho_upload,&b,&n); } /* no chain after all: the per-matrix path's */
if(g_k3_dho) k3_dho_place(m); /* then the host copies go, before the tier sizes its budget */
if(cuda_on) fprintf(stderr,"[VK] tier kimi_k3: the CUDA expert tier is on and wins; the Vulkan tier stays off\n");
else if(nmoe&&vkt_wanted()){
VktConfig vc={.engine="kimi_k3",.layers=c->n_layers,.experts=c->n_experts,
.hidden=c->latent,.inter=c->moe_inter,.topk=c->topk,
.gate_up={VKT_SRC_MXFP4_E8M0,32},.down={VKT_SRC_MXFP4_E8M0,32},
.act=VKT_ACT_SITU,.act_a=c->situ_b1,.act_b=c->situ_b2,
.max_rows=K3_VK_ROWS*c->topk,
.ram_reserve=(size_t)(m->e_slot+8192)*(size_t)cap*(size_t)nmoe,
.dense_bytes=coli_vk_dense()&&!g_k3c_on&&!g_k3_dho&&!g_k3_partial?k3_vk_dense_bytes(m):0, /* the chain's are there already; a partial chain: nothing more goes up */
.in_ram=k3_vk_in_ram,.ram_ctx=m,
.load=k3_vk_load,.release=k3_vk_unhold,.load_ctx=m,.load_batch=k3_vk_load_batch};
atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if(vkt_init(&vc,rt_counts_all())){
atexit(vkt_shutdown);
int all=c->n_layers*c->n_experts;
int *pl=malloc((size_t)all*sizeof(int)), *pe=malloc((size_t)all*sizeof(int));
const char *warm=getenv("COLI_VK_TIER_WARM"); /* 0: no warm start, the tier fills as experts pass by */
int n=pl&&pe&&!(warm&&*warm=='0')?vkt_plan(pl,pe,all):0;
if(n>0){
double t0=now_s();
#pragma omp parallel for schedule(dynamic,1)
for(int i=0;i<n;i++){
Slot tmp; memset(&tmp,0,sizeof tmp); tmp.eid=-1;
if(!k3_expert_on_disk(m,pl[i],pe[i])){ vkt_put(pl[i],pe[i],NULL); continue; }
expert_read(m,pl[i],pe[i],&tmp);
VktExpertSrc src=k3_vk_src(m,tmp.buf);
vkt_put(pl[i],pe[i],&src);
free(tmp.base);
}
vkt_put_done();
fprintf(stderr,"[VK] tier kimi_k3: warm start, %d experts from the history in %.1fs\n",n,now_s()-t0);
}
free(pl); free(pe);
}
}
if(!vkt_ready()&&!coli_vk_dense()) coli_vk_dense_decide("kimi_k3",0,1); /* no tier after all */
if(coli_vk_dense()){
k3c_adopt_shared(m); /* the chain's copies, where it made them */
int nsh=0;
for(int i=0;i<c->n_layers;i++){
if(!m->L[i].sparse) continue;
Moe *o=&m->L[i].moe;
if(g_k3_partial) /* a partial chain: nothing new goes up, the CPU's layers keep theirs */
nsh+=(o->sh_gate.vk!=NULL)+(o->sh_up.vk!=NULL)+(o->sh_down.vk!=NULL);
else nsh+=w_vk_upload(&o->sh_gate)+w_vk_upload(&o->sh_up)+w_vk_upload(&o->sh_down);
}
size_t used=0, count=0; coli_vk_mem_info(&used,&count);
fprintf(stderr,"[VK] kimi_k3: %d shared-expert matrices on the device (%.2f MiB)\n",nsh,used/1048576.0);
}
k3c_atexit(); /* after the tier's: the chain goes before the device */
}
static void k3_vk_report(const Model *m, const char *scope){
if(!g_k3_vk) return;
fprintf(stderr,"[VK] kimi_k3: %llu matmuls on the GPU\n",coli_vk_matmul_calls());
{ size_t bytes=0, tensors=0; coli_vk_mem_info(&bytes,&tensors);
fprintf(stderr,"[VK] kimi_k3: %zu resident matrices on the device, %.1f MiB\n",tensors,bytes/1048576.0); }
vkt_report(scope,m->hits,m->miss);
k3c_report(); /* the dense chain's line, when it ran */
}
#endif
/* The two fused shared experts at full width, [C, hidden] (falloc'd). */
static float *k3_shared(Model *m, Moe *o, const float *x, int C){
Cfg *c=&m->c;
int shi=c->moe_inter*c->n_shared;
float *sg=falloc((int64_t)C*shi), *su=falloc((int64_t)C*shi), *sd=falloc((int64_t)C*c->hidden);
w_matmul(sg,x,&o->sh_gate,C); w_matmul(su,x,&o->sh_up,C);
for(int64_t i=0;i<(int64_t)C*shi;i++) sg[i]=situf_(sg[i],su[i],c->situ_b1,c->situ_b2);
w_matmul(sd,sg,&o->sh_down,C);
free(sg); free(su);
return sd;
}
#ifdef COLI_VULKAN
/* The tier's streaming (a big prompt chunk's cold experts on the device): a group of
* experts as experts_apply_union resolves them -- the layer cache, the misses read into
* the working-set slots in parallel -- each valid until the tier gives it back; the
* misses then join the LRU as experts_apply_union's do, when the group's last one is
* given back (before anything else can reuse the working set). */
static int g_k3s_layer=-1, g_k3s_nmiss=0, g_k3s_held=0;
static void k3_vk_promote(Model *m){
int li=g_k3s_layer, nmiss=g_k3s_nmiss;
g_k3s_layer=-1; g_k3s_nmiss=0; g_k3s_held=0;
if(li<0 || nmiss<=0) return;
LCache *lc=&m->ecache[li];
int promo = nmiss<lc->cap ? nmiss : lc->cap;
for(int a=0;a<promo;a++){
int q=nmiss-1-a; Slot *dst;
if(lc->n<lc->cap) dst=&lc->s[lc->n++];
else {
int lru=-1;
for(int i=0;i<lc->n;i++){
if(lc->s[i].pinned) continue;
if(lru<0 || lc->s[i].used<lc->s[lru].used) lru=i;
}
if(lru<0) break;
dst=&lc->s[lru];
}
cache_promote(m,li,dst,&m->ws[q]);
}
}
static int k3_vk_load_batch(void *ctx,int li,const int *e,int n,VktExpertSrc *srcs,void **h){
Model *m=ctx;
k3_vk_promote(m);
if(n<1) return 0;
if(n>LP_MAX) n=LP_MAX;
if(n>64) n=64;
Slot *use[64]; int miss[64], nmiss=0;
for(int j=0;j<n;j++){
ehit_mark(m,li,e[j]);
use[j]=slot_find(m,li,e[j]);
if(!use[j]){ m->miss++; use[j]=&m->ws[nmiss]; miss[nmiss++]=j; }
}
if(nmiss){
double t0=now_s();
#pragma omp parallel for schedule(dynamic,1)
for(int q=0;q<nmiss;q++) expert_read(m,li,e[miss[q]],&m->ws[q]);
m->t_eload+=now_s()-t0;
m->ebytes+=(uint64_t)nmiss*(uint64_t)m->e_slot;
}
for(int j=0;j<n;j++){ srcs[j]=k3_vk_src(m,use[j]->buf); h[j]=m; }
g_k3s_layer=li; g_k3s_nmiss=nmiss; g_k3s_held=n;
return n;
}
static int k3_vk_load(void *ctx,int li,int e,VktExpertSrc *src,void **h){
return k3_vk_load_batch(ctx,li,&e,1,src,h)==1;
}
static void k3_vk_unhold(void *ctx,void *h){ (void)h; if(g_k3s_held>0 && --g_k3s_held==0) k3_vk_promote((Model *)ctx); }
/* ---- the shared Vulkan routed-expert tier, one MoE layer step -------------
* k3_vk_issue hands the step's (position, rank) pairs whose expert is resident
* on the device to one asynchronous batch (vkt_issue) as soon as z is known.
* The CPU then reads and computes the other pairs (experts_apply_union in rows
* mode: an expert the device took entirely is never read from disk) and the
* shared experts while the batch runs; after the join every pair's output
* joins its position in the order the CPU-only run adds them -- the union's
* order (disk offset), the device's rows and the CPU's alike -- so the order of
* the sum never depends on which experts were resident, and an expert computed
* on the CPU keeps the CPU run's bits. The experts the CPU computed are offered
* to the tier (vkt_note) after the join, those still in the RAM cache: a
* promotion that displaces a resident then frees it at once, and its upload
* starts on a pool with room instead of waiting on the uploader thread for the
* join's free. */
typedef struct { int *idx, *left; uint8_t *taken; const float **dev; int ndev; } K3VkStep;
static void k3_vk_issue(Model *m, int li, int C, const int *keff, const int *idxs,
const float *z, K3VkStep *st){
int K=m->c.topk, E=m->c.n_experts, n=C*K;
st->idx=malloc((size_t)n*sizeof(int)); st->taken=malloc((size_t)n);
st->dev=malloc((size_t)n*sizeof(*st->dev)); st->left=calloc((size_t)E,sizeof(int));
if(!st->idx||!st->taken||!st->dev||!st->left){ fprintf(stderr,"OOM vk step\n"); exit(1); }
for(int t=0;t<C;t++) for(int kk=0;kk<K;kk++)
st->idx[(int64_t)t*K+kk]=kk<keff[t]?idxs[(int64_t)t*K+kk]:-1;
st->ndev=vkt_issue(li,z,C,K,st->idx,st->taken);
for(int i=0;i<n;i++){
if(st->idx[i]<0) continue;
if(st->taken[i]) ehit_mark(m,li,st->idx[i]);
else st->left[st->idx[i]]++; /* pairs of this expert the CPU computes */
}
}
static void k3_vk_step_free(K3VkStep *st){
free(st->idx); free(st->left); free(st->taken); free(st->dev);
memset(st,0,sizeof(*st));
}
/* The entries of the union lists whose pair is (taken == want), as lists of their
* own in the same order; rslot = the entry's index in the full lists. */
static int k3_vk_sublist(int nu, const int *uid, const int *pfirst, const int *pcnt,
const int *poslist, const float *wlist, const int *epair,
const uint8_t *taken, int want,
int *u2, int *f2, int *c2, int *p2, float *w2, int *s2){
int n2=0, at=0;
for(int j=0;j<nu;j++){
int start=at;
for(int q=0;q<pcnt[j];q++){
int e=pfirst[j]+q;
if(!!taken[epair[e]]!=want) continue;
p2[at]=poslist[e]; w2[at]=wlist[e]; s2[at]=e; at++;
}
if(at>start){ u2[n2]=uid[j]; f2[n2]=start; c2[n2]=at-start; n2++; }
}
return n2;
}
/* The step after k3_vk_issue: the CPU's pairs, the shared experts (returned),
* the join, and the sum into u in the CPU run's order. */
static float *k3_vk_step(Model *m, Moe *o, int li, int C, const float *x, const float *z,
float *u, int nu, const int *uid, const int *pfirst, const int *pcnt,
const int *poslist, const float *wlist,
float *gate, float *up, float *hz, K3VkStep *st){
Cfg *c=&m->c; int K=c->topk, LT=c->latent;
int tot=0;
for(int j=0;j<nu;j++) tot+=pcnt[j];
int *epair=malloc((size_t)tot*sizeof(int));
int *u2=malloc((size_t)nu*sizeof(int)), *f2=malloc((size_t)nu*sizeof(int)), *c2=malloc((size_t)nu*sizeof(int));
int *p2=malloc((size_t)tot*sizeof(int)), *s2=malloc((size_t)tot*sizeof(int));
float *w2=falloc(tot), *R=falloc((int64_t)tot*LT);
if(!epair||!u2||!f2||!c2||!p2||!s2){ fprintf(stderr,"OOM vk step\n"); exit(1); }
for(int j=0;j<nu;j++) for(int q=0;q<pcnt[j];q++){ /* entry -> its (position, rank) */
int e=pfirst[j]+q, t=poslist[e], kk=0;
while(kk<K-1&&st->idx[(int64_t)t*K+kk]!=uid[j]) kk++;
epair[e]=t*K+kk;
}
int n2=k3_vk_sublist(nu,uid,pfirst,pcnt,poslist,wlist,epair,st->taken,0,u2,f2,c2,p2,w2,s2);
if(n2) experts_apply_union(m,li,n2,u2,f2,c2,p2,w2,z,LT,u,gate,up,hz,R,s2);
float *sd=x?k3_shared(m,o,x,C):NULL; /* while the batch runs (x NULL: the dense chain's) */
if(st->ndev&&!vkt_join(st->dev)){ /* the batch failed: those pairs here */
n2=k3_vk_sublist(nu,uid,pfirst,pcnt,poslist,wlist,epair,st->taken,1,u2,f2,c2,p2,w2,s2);
if(n2) experts_apply_union(m,li,n2,u2,f2,c2,p2,w2,z,LT,u,gate,up,hz,R,s2);
memset(st->taken,0,(size_t)C*K);
} else
for(int j=0;j<nu;j++){ /* the CPU's experts, nothing in flight */
Slot *sl=st->left[uid[j]]?slot_indexed(m,li,uid[j]):NULL;
if(!sl) continue;
VktExpertSrc vs=k3_vk_src(m,sl->buf);
vkt_note(li,uid[j],&vs);
}
for(int j=0;j<nu;j++) for(int q=0;q<pcnt[j];q++){
int e=pfirst[j]+q, t=poslist[e], pr=epair[e];
const float *row=st->taken[pr]?st->dev[pr]:R+(int64_t)e*LT;
float *ut=u+(int64_t)t*LT, wk=wlist[e];
for(int i=0;i<LT;i++) ut[i]+=wk*row[i];
}
free(epair); free(u2); free(f2); free(c2); free(p2); free(s2); free(w2); free(R);
return sd;
}
#endif
/* The router's choice for C rows from their raw scores (sco [C][E], turned into
* sigmoids in place): the top-k with the correction bias, the weights renormalized,
* K3_TOPP; idxs and wsels [C][K], keff[C] the ranks kept. */
static void k3_moe_route(Model *m, Moe *o, int li, float *sco, int C, int *idxs, float *wsels, int *keff){
Cfg *c=&m->c;
int E=c->n_experts, K=c->topk;
for(int t=0;t<C;t++){
float *st=sco+(int64_t)t*E;
for(int e=0;e<E;e++) st[e]=sigmoidf_(st[e]);
int *idx=idxs+(int64_t)t*K; float *wsel=wsels+(int64_t)t*K;
for(int kk=0;kk<K;kk++){
int best=-1; float bv=-1e30f;
for(int e=0;e<E;e++){
int taken=0; for(int j=0;j<kk;j++) if(idx[j]==e){taken=1;break;}
float sv=st[e]+o->rbias[e];
if(!taken&&sv>bv){ bv=sv; best=e; }
}
/* SEC: all-NaN scores leave best at -1, and st[-1] is read on the
* very next expression. See rt_router_pick in route_trace.h. */
best = rt_router_pick(best, kk, E, li);
idx[kk]=best; wsel[kk]=st[best]; /* weight = RAW sigmoid score */
}
{ float sm=0; for(int kk=0;kk<K;kk++) sm+=wsel[kk];
for(int kk=0;kk<K;kk++) wsel[kk]/=(sm+1e-20f); }
int Kt=K;
/* K3_TOPP: drop the low-weight tail — the only lever that cuts expert
* I/O AND compute proportionally. Weights renormalize over the kept
* set (GLM's TOPP semantics). Quality-gate via K3_LOGITS. */
if(g_k3_topp>0.f && g_k3_topp<1.f){
for(int a2=1;a2<Kt;a2++){ int e=idx[a2]; float w2=wsel[a2]; int b2=a2-1;
while(b2>=0&&wsel[b2]<w2){ idx[b2+1]=idx[b2]; wsel[b2+1]=wsel[b2]; b2--; }
idx[b2+1]=e; wsel[b2+1]=w2; }
float cum=0; int keep=Kt;
for(int kk=0;kk<Kt;kk++){ cum+=wsel[kk]; if(cum>=g_k3_topp){ keep=kk+1; break; } }
if(keep<Kt){
float sm=0; for(int kk=0;kk<keep;kk++) sm+=wsel[kk];
for(int kk=0;kk<keep;kk++) wsel[kk]/=(sm+1e-20f);
Kt=keep;
}
}
keff[t]=Kt;
rt_route(li,t,idx,wsel,Kt); /* traces and counts, one call */
}
}
/* The routed experts of C rows in the latent: u[C][LT] (zeroed by the caller) gets
* every kept rank's weighted output, in the union's order. x: the rows the shared
* experts read while a tier batch runs (NULL: the dense chain computes them on the
* device); their output is returned when this step computed it, else NULL. */
static float *k3_moe_experts(Model *m, Moe *o, int li, const float *x, const float *z, int C,
const int *keff, const int *idxs, const float *wsels, float *u){
Cfg *c=&m->c;
int E=c->n_experts, K=c->topk, LT=c->latent, MI=c->moe_inter;
float *gate=falloc(MI), *up=falloc(MI), *hz=falloc(LT), *sd=NULL;
#ifdef COLI_VULKAN
K3VkStep vst={0}; /* the device's pairs, issued before the union is read */
if(vkt_ready()) k3_vk_issue(m,li,C,keff,idxs,z,&vst);
#endif
/* union across the chunk: each unique expert loads ONCE and applies to
* every position that selected it (position lists via counting sort).
* With QB-flat routing the dedup is modest (~15% at C=32), but the loads
* arrive as one deep burst for the NVMe and the dense side above/below
* batches perfectly. */
{
int *map=malloc((size_t)E*sizeof(int));
int *uid=malloc((size_t)C*K*sizeof(int));
int *pcnt=malloc((size_t)C*K*sizeof(int)), *pfirst=malloc((size_t)C*K*sizeof(int));
int *poslist=malloc((size_t)C*K*sizeof(int)); float *wlist=falloc((int64_t)C*K);
int *cur=malloc((size_t)C*K*sizeof(int));
if(!map||!uid||!pcnt||!pfirst||!poslist||!cur){fprintf(stderr,"OOM moe union\n");exit(1);}
for(int e=0;e<E;e++) map[e]=-1;
int nu=0;
for(int t=0;t<C;t++) for(int kk=0;kk<keff[t];kk++){
int e=idxs[(int64_t)t*K+kk];
if(map[e]<0){ map[e]=nu; uid[nu]=e; pcnt[nu]=0; nu++; }
pcnt[map[e]]++;
}
int acc=0;
for(int j=0;j<nu;j++){ pfirst[j]=acc; cur[j]=acc; acc+=pcnt[j]; }
for(int t=0;t<C;t++) for(int kk=0;kk<keff[t];kk++){
int j=map[idxs[(int64_t)t*K+kk]];
poslist[cur[j]]=t; wlist[cur[j]]=wsels[(int64_t)t*K+kk]; cur[j]++;
}
/* keep loads in DISK-OFFSET order (experts are NOT id-ordered inside
* the HF shards — measured 169/895); permute the list heads with the
* ids. WILLNEED prefetch only for the buffered path. */
for(int a2=0;a2<nu-1;a2++) for(int b2=a2+1;b2<nu;b2++){
ERef *ea=&m->eref[(int64_t)li*E+uid[a2]], *eb=&m->eref[(int64_t)li*E+uid[b2]];
if(eb->fd[0]<ea->fd[0]||(eb->fd[0]==ea->fd[0]&&eb->off[0]<ea->off[0])){
int tt=uid[a2];uid[a2]=uid[b2];uid[b2]=tt;
tt=pcnt[a2];pcnt[a2]=pcnt[b2];pcnt[b2]=tt;
tt=pfirst[a2];pfirst[a2]=pfirst[b2];pfirst[b2]=tt; }
}
if(!g_k3_direct)
for(int j=0;j<nu;j++){
#ifdef COLI_VULKAN
if(vst.ndev&&!vst.left[uid[j]]) continue; /* the device computes it: nothing to read */
#endif
ERef *er=&m->eref[(int64_t)li*E+uid[j]];
int64_t sizes[6]={m->e_w1p,m->e_w1s,m->e_w2p,m->e_w2s,m->e_w1p,m->e_w1s};
if(er->contig){ if(er->fd[0]>=0) posix_fadvise(er->fd[0],er->off[0],m->e_slot,POSIX_FADV_WILLNEED); }
else for(int k2=0;k2<6;k2++) if(er->fd[k2]>=0) posix_fadvise(er->fd[k2],er->off[k2],sizes[k2],POSIX_FADV_WILLNEED);
}
#ifdef COLI_VULKAN
if(vst.ndev) sd=k3_vk_step(m,o,li,C,x,z,u,nu,uid,pfirst,pcnt,poslist,wlist,gate,up,hz,&vst);
else
#endif
experts_apply_union(m,li,nu,uid,pfirst,pcnt,poslist,wlist,z,LT,u,gate,up,hz,NULL,NULL);
free(map);free(uid);free(pcnt);free(pfirst);free(poslist);free(wlist);free(cur);
}
#ifdef COLI_VULKAN
k3_vk_step_free(&vst);
#endif
free(gate);free(up);free(hz);
return sd;
}
static void moe_forward(Model *m, Layer *l, int li, const float *x, int C, float *out){
Cfg *c=&m->c; Moe *o=&l->moe;
int E=c->n_experts, K=c->topk, LT=c->latent;
float *sco=falloc((int64_t)C*E);
if(!K3_F32_DEV(o->vk_router,sco,x,C,c->hidden,E)){ K3_F32_NEED(o->router); matmul(sco,x,o->router,C,c->hidden,E); }
int *idxs=malloc((size_t)C*K*sizeof(int)); float *wsels=falloc((int64_t)C*K);
int *keff=malloc((size_t)C*sizeof(int));
if(!idxs||!keff){fprintf(stderr,"OOM moe sel\n");exit(1);}
k3_moe_route(m,o,li,sco,C,idxs,wsels,keff);
float *z=falloc((int64_t)C*LT), *u=falloc((int64_t)C*LT);
w_matmul(z,x,&o->lat_down,C);
memset(u,0,(size_t)C*LT*sizeof(float));
float *sd=k3_moe_experts(m,o,li,x,z,C,keff,idxs,wsels,u);
for(int t=0;t<C;t++)
rmsnorm_(u+(int64_t)t*LT,u+(int64_t)t*LT,o->lat_norm,LT,c->eps);
w_matmul(out,u,&o->lat_up,C);
/* shared experts at full width (a tier step computed them while its batch ran) */
if(!sd) sd=k3_shared(m,o,x,C);
for(int64_t d=0;d<(int64_t)C*c->hidden;d++) out[d]+=sd[d];
free(sco);free(idxs);free(wsels);free(keff);
free(z);free(u);free(sd);
}
static void dense_forward(Model *m, Layer *l, const float *x, int C, float *out){
Cfg *c=&m->c; int DI=c->dense_inter;
float *g=falloc((int64_t)C*DI), *u=falloc((int64_t)C*DI);
w_matmul(g,x,&l->d_gate,C); w_matmul(u,x,&l->d_up,C);
for(int64_t i=0;i<(int64_t)C*DI;i++) g[i]=situf_(g[i],u[i],c->situ_b1,c->situ_b2);
w_matmul(out,g,&l->d_down,C);
free(g);free(u);
}
/* ---------- a CHUNK of C tokens through the stack, layer-major: every dense
* matmul batches over the chunk (weights stream from RAM once per chunk), the
* MoE loads each unique expert once. Sequential state (KDA recurrence, MLA
* cache, AttnRes bookkeeping) advances per token inside each layer, which is
* exactly the original order — chunked results are bit-identical to C=1.
* Returns the LAST position's logits (falloc'd), or NULL pre-head. ---------- */
static float *g_x0=NULL; static int g_x0_n=0; /* K3_X0: injected inputs (validation) */
static FILE *g_lfp=NULL; /* K3_LOGITS: per-position logit dump */
typedef int (*K3CancelPoll)(void *context);
/* Run an arbitrary contiguous layer range on boundary activations. Kimi's
* AttnRes block residuals are part of that boundary state: hidden alone is
* insufficient whenever a range begins after layer zero. `bres` therefore
* contains nbmax hidden-width rows per token and nb_io names the live prefix. */
static int k3_layers_forward_range(Model *m, float *hidden, float *bres,
int *nb_io, int pos0, int C,
int layer_begin, int layer_end,
K3CancelPoll poll_cancel,
void *cancel_context, int *cancelled){
Cfg *c=&m->c; int D=c->hidden;
int nbmax=(c->n_layers+c->res_bs-1)/c->res_bs;
int nb=*nb_io;
float *prefix=falloc((int64_t)C*D), *nrm=falloc((int64_t)C*D);
float *att=falloc((int64_t)C*D), *mix=falloc(D), *mlp=falloc((int64_t)C*D);
if(cancelled) *cancelled=0;
for(int i=layer_begin;i<layer_end;i++){
Layer *l=&m->L[i];
int snap=(i%c->res_bs==0); /* block boundary: same for all t */
if(snap&&nb>=nbmax){
if(cancelled) *cancelled=1;
break;
}
for(int t=0;t<C;t++){
float *h=hidden+(int64_t)t*D, *p=prefix+(int64_t)t*D;
memcpy(p,h,D*sizeof(float)); /* prefix_sum at entry */
if(nb>0) res_mix(h,p,bres+(int64_t)t*nbmax*D,nb,D,l->attn_sw,c->eps);
if(snap) memcpy(bres+(int64_t)t*nbmax*D+(int64_t)nb*D,p,D*sizeof(float));
rmsnorm_(nrm+(int64_t)t*D,h,l->in_ln,D,c->eps);
/* Phase 9: capture attn-input nrm for target token */
if(g_k3_val_layer==i && t==g_k3_val_token){
val_dump("nrm_attn",nrm+(int64_t)t*D,D); }
}
int have_prefix=!snap;
if(snap) nb++;
double t0=now_s();
if(l->kda) kda_forward(m,l,i,nrm,C,att);
else mla_forward(m,l,i,nrm,pos0,C,att);
{ static int once=0; if(i==0 && !once){ once=1;
fprintf(stderr,"[K3-PATH] L0 attn=%s metal=%d\n", l->kda?"KDA":"MLA", g_k3_metal); fflush(stderr); } }
m->t_attn+=now_s()-t0;
/* Phase 9: capture attn output (att) for target token */
if(g_k3_val_layer==i && g_k3_val_token<C) val_dump("att",att+(int64_t)g_k3_val_token*D,D);
for(int t=0;t<C;t++){
float *p=prefix+(int64_t)t*D, *a=att+(int64_t)t*D;
if(have_prefix){ for(int d=0;d<D;d++) p[d]+=a[d]; }
else { memcpy(p,a,D*sizeof(float)); }
res_mix(mix,p,bres+(int64_t)t*nbmax*D,nb,D,l->mlp_sw,c->eps);
rmsnorm_(nrm+(int64_t)t*D,mix,l->post_ln,D,c->eps);
/* Phase 9: capture MLP-input (post-attention nrm) */
if(g_k3_val_layer==i && t==g_k3_val_token){
val_dump("nrm_mlp",nrm+(int64_t)t*D,D); }
}
t0=now_s();
if(l->sparse) moe_forward(m,l,i,nrm,C,mlp);
else dense_forward(m,l,nrm,C,mlp);
m->t_moe+=now_s()-t0;
/* Phase 9: capture mlp output for target token */
if(g_k3_val_layer==i && g_k3_val_token<C) val_dump("mlp",mlp+(int64_t)g_k3_val_token*D,D);
for(int t=0;t<C;t++){
float *p=prefix+(int64_t)t*D;
for(int d=0;d<D;d++) p[d]+=mlp[(int64_t)t*D+d];
memcpy(hidden+(int64_t)t*D,p,D*sizeof(float));
if(m->trace) fwrite(hidden+(int64_t)t*D,sizeof(float),D,m->trace);
/* Phase 9: capture layer output (hidden) for target token */
if(g_k3_val_layer==i && t==g_k3_val_token){
val_dump("hidden",hidden+(int64_t)t*D,D); }
}
/* Prefill does not emit tokens, and a full chunk can take minutes on
* CPU. Poll after each layer so the gateway can cancel before the
* first token instead of waiting for the entire prompt. */
if(poll_cancel&&poll_cancel(cancel_context)){
if(cancelled) *cancelled=1;
break;
}
}
*nb_io=nb;
free(prefix);free(nrm);free(att);free(mix);free(mlp);
return cancelled&&*cancelled?-1:0;
}
/* Canale logprobs (modalita jev). Anche qui la fotografia dello stato c'e
* gia: si chiama K3Ckpt e salva kstate piu le tre finestre di convoluzione di
* ogni strato KDA. Mancava la lettura del prefill e, alla fotografia, i logit
* finali: senza quelli il primo token fresco di ogni opzione resta senza
* predittore.
*
* Il prefill qui va a blocchi (K3_CHUNK), quindi la posizione che predice il
* primo token di un blocco sta nel blocco PRECEDENTE. Il riporto tiene quella
* riga di logit fra un blocco e l'altro; all'inizio lo riempie la fotografia. */
static int g_echo_k = 0;
static const char *g_echo_id = NULL;
static Tok *g_echo_tok = NULL;
static float *g_echo_carry = NULL; /* logit dell'ultima posizione gia fatta */
static int g_echo_carry_ok = 0;
static void k3_echo(const char *id, int pos, int token, const float *lo, int V, int k){
char tail[1024]; coli_logprob_tail(tail, sizeof tail, lo, V, token, k);
char piece[512]; int n = g_echo_tok ? tok_decode(g_echo_tok, &token, 1, piece, (int)sizeof piece) : 0;
if(n<0) n=0;
printf("ECHO %s %d %d%s\n", id, n, pos, tail);
if(n>0) fwrite(piece,1,(size_t)n,stdout);
fputc('\n', stdout); fflush(stdout);
}
/* The input rows of positions pos0..pos0+C-1: the embeddings of ids, or K3_X0's rows. */
static void k3_embed(Model *m, const int *ids, int pos0, int C, float *hidden){
int D=m->c.hidden;
for(int t=0;t<C;t++){
if(g_x0){
if(pos0+t>=g_x0_n){ fprintf(stderr,"K3_X0: pos %d beyond %d injected rows\n",pos0+t,g_x0_n); exit(1); }
memcpy(hidden+(int64_t)t*D,g_x0+(int64_t)(pos0+t)*D,D*sizeof(float));
} else {
char nm[512]; snprintf(nm,sizeof(nm),"%smodel.embed_tokens.weight",m->pfx);
st_read_slice_f32(&m->S,nm,(int64_t)ids[t]*D,D,hidden+(int64_t)t*D,0);
}
}
}
#ifdef COLI_VULKAN
#include "kimi_k3_chain.h" /* COLI_VK_CHAIN: every layer's dense chain on the device */
#endif
static float *step_chunk_ex(Model *m, const int *ids, int pos0, int C,
K3CancelPoll poll_cancel, void *cancel_context,
int *cancelled){
Cfg *c=&m->c; int D=c->hidden;
if(cancelled) *cancelled=0;
int nbmax=(c->n_layers+c->res_bs-1)/c->res_bs;
float *hidden=falloc((int64_t)C*D), *bres=falloc((int64_t)C*nbmax*D);
float *mix=falloc(D);
int nb=0;
k3_embed(m,ids,pos0,C,hidden);
if(pos0==0&&C<=1) fprintf(stderr,"[DBG] step_chunk pos=%d C=%d metal=%d\n", pos0, C, g_k3_metal);
int echo_on = g_echo_k>0 && g_echo_id;
int from=0; /* the first layer the CPU runs */
#ifdef COLI_VULKAN
/* the layers on the device (kimi_k3_chain.h). Every layer: the final rows (fin, when
* the head reads more than the last one) and the last row's logits (dev_last) come
* back. A partial chain's first N: the rows' AttnRes state after layer N - 1 comes
* back in hidden, bres and nb, and the CPU runs the rest from there. 0: the CPU runs
* every layer below, its state current first */
float *fin=NULL, *dev_last=NULL; int chain=0;
if(g_k3c_on){
chain=k3c_forward(m,hidden,bres,&nb,pos0,C,g_lfp||g_k3_val_lfp||echo_on,&fin,&dev_last,
poll_cancel,cancel_context,cancelled);
if(!chain) k3c_cpu_step(m,pos0);
}
from=chain;
#endif
if(from<c->n_layers&&!(cancelled&&*cancelled))
k3_layers_forward_range(m,hidden,bres,&nb,pos0,C,from,c->n_layers,
poll_cancel,cancel_context,cancelled);
float *logits=NULL;
if((!cancelled||!*cancelled)&&m->has_head){
double t0=now_s();
for(int t=0;t<C;t++){
/* head only where needed: the chunk's last token (feeds sampling)
* and every position when K3_LOGITS dumps teacher-forced logits;
* also all positions during Phase 10 validation; and every
* position when a request asked for the logprobs channel */
if(!g_lfp && !g_k3_val_lfp && !echo_on && t<C-1) continue;
float *lo=falloc(c->vocab);
#ifdef COLI_VULKAN
if(chain==c->n_layers){ /* the chain's final rows, normalized; the last row's logits from the device */
if(t==C-1&&dev_last) memcpy(lo,dev_last,(size_t)c->vocab*sizeof(float));
else w_matmul(lo,fin+(int64_t)t*D,&m->lm_head,1);
} else
#endif
{
res_mix(mix,hidden+(int64_t)t*D,bres+(int64_t)t*nbmax*D,nb,D,m->out_sw,c->eps);
rmsnorm_(mix,mix,m->final_norm,D,c->eps);
if(m->trace) fwrite(mix,sizeof(float),D,m->trace);
w_matmul(lo,mix,&m->lm_head,1);
}
if(g_lfp) fwrite(lo,sizeof(float),(size_t)c->vocab,g_lfp);
if(g_k3_val_lfp) fwrite(lo,sizeof(float),(size_t)c->vocab,g_k3_val_lfp);
if(echo_on){
if(t==0) k3_echo(g_echo_id,pos0,ids[0],
g_echo_carry_ok?g_echo_carry:NULL,c->vocab,g_echo_k);
if(t+1<C) k3_echo(g_echo_id,pos0+t+1,ids[t+1],lo,c->vocab,g_echo_k);
else if(g_echo_carry){
memcpy(g_echo_carry,lo,(size_t)c->vocab*sizeof(float));
g_echo_carry_ok=1;
}
}
if(t==C-1) logits=lo; else free(lo);
}
m->t_head+=now_s()-t0;
}
/* record what was just fed, at the positions it went to (kv_prefix.h) */
if(!cancelled||!*cancelled) kv_prefix_record(&m->kvp,ids,pos0,C);
free(hidden);free(bres);free(mix);
#ifdef COLI_VULKAN
free(fin);free(dev_last);
#endif
return logits;
}
static float *step_chunk(Model *m, const int *ids, int pos0, int C){
return step_chunk_ex(m,ids,pos0,C,NULL,NULL,NULL);
}
/* ---- several conversations at once (KV_SLOTS>1, serve_mux) ------------------- */
static void k3_seq_swap(Model *m, K3Seq *q){
#define K3_SEQ_SWAP(T,a,b) do{ T t_=(a); (a)=(b); (b)=t_; }while(0)
K3_SEQ_SWAP(float**,m->kstate,q->kstate); K3_SEQ_SWAP(float**,m->cwq,q->cwq);
K3_SEQ_SWAP(float**,m->cwk,q->cwk); K3_SEQ_SWAP(float**,m->cwv,q->cwv);
K3_SEQ_SWAP(float**,m->Lc,q->Lc); K3_SEQ_SWAP(float**,m->Rc,q->Rc);
K3_SEQ_SWAP(int,m->max_t,q->max_t); K3_SEQ_SWAP(kv_prefix,m->kvp,q->kvp);
if(!q->Ic&&!(q->Ic=calloc(m->c.n_layers,sizeof(float*)))){ fprintf(stderr,"[serve] out of memory\n"); exit(1); } /* slot 0's place */
for(int i=0;i<m->c.n_layers;i++) K3_SEQ_SWAP(float*,m->L[i].m.Ic,q->Ic[i]);
#undef K3_SEQ_SWAP
}
/* A conversation's recurrent state of its own; its MLA caches come with its first
* request (prepare_request_state's kv_alloc), as the Model's do. */
static void k3_seq_alloc(Model *m, K3Seq *q){
Cfg *c=&m->c; int P=c->kda_proj;
memset(q,0,sizeof *q);
q->kstate=calloc(c->n_layers,sizeof(float*)); q->cwq=calloc(c->n_layers,sizeof(float*));
q->cwk=calloc(c->n_layers,sizeof(float*)); q->cwv=calloc(c->n_layers,sizeof(float*));
q->Ic=calloc(c->n_layers,sizeof(float*));
for(int i=0;i<c->n_layers;i++) if(m->L[i].kda&&m->kstate[i]){
q->kstate[i]=afcalloc((int64_t)c->kda_heads*c->kda_hd*c->kda_hd);
q->cwq[i]=afcalloc((int64_t)P*c->conv_k);
q->cwk[i]=afcalloc((int64_t)P*c->conv_k);
q->cwv[i]=afcalloc((int64_t)P*c->conv_k);
}
}
/* One decode step of several conversations: row s is the token ids[s] at
* rows[s].pos of the conversation rows[s].seq, every conversation parked. The
* matrices, the routed experts and lm_head run once over the S rows; the KDA and MLA
* read and write each row's own state (m->mux_rows). The CPU kernels give a row the
* same bits whatever S is, so each conversation gets the logits it would alone. */
static float *k3_step_rows(Model *m, const K3Row *rows, const int *ids, int S){
Cfg *c=&m->c; int D=c->hidden;
int nbmax=(c->n_layers+c->res_bs-1)/c->res_bs, nb=0;
float *hidden=falloc((int64_t)S*D), *bres=falloc((int64_t)S*nbmax*D);
k3_embed(m,ids,0,S,hidden);
m->mux_rows=rows;
k3_layers_forward_range(m,hidden,bres,&nb,0,S,0,c->n_layers,NULL,NULL,NULL);
m->mux_rows=NULL;
float *mix=falloc((int64_t)S*D), *logits=falloc((int64_t)S*c->vocab);
double t0=now_s();
for(int t=0;t<S;t++){
res_mix(mix+(int64_t)t*D,hidden+(int64_t)t*D,bres+(int64_t)t*nbmax*D,nb,D,m->out_sw,c->eps);
rmsnorm_(mix+(int64_t)t*D,mix+(int64_t)t*D,m->final_norm,D,c->eps);
}
w_matmul(logits,mix,&m->lm_head,S);
m->t_head+=now_s()-t0;
for(int t=0;t<S;t++) kv_prefix_record(&rows[t].seq->kvp,ids+t,rows[t].pos,1);
free(hidden); free(bres); free(mix);
return logits;
}
static void kv_alloc(Model *m, int max_t){
Cfg *c=&m->c;
/* Serve calls this once per request, and it used to calloc Lc/Rc over the
* old pointers without freeing them: n_layers x max_t x (kv_lora +
* qk_rope) floats leaked every turn, hundreds of MB over a conversation on
* the 24 MLA layers at 4k context.
*
* GROW, DO NOT RESTART. Freeing and re-allocating also discards every
* position already computed, which defeats KV prefix reuse in the one case
* it exists for: a conversation whose prompt is longer every turn asks for
* a larger max_t every turn, so the state would be thrown away immediately
* before the point of using it. (Caught by CI on the Inkling side, where
* the same shape of bug sat in the same place.)
*
* Lc/Rc are laid out [position][kv_lora] and [position][qk_rope], so unlike
* inkling's head-major K/V a grow is a straight prefix copy — no re-layout.
* The 69 KDA layers are untouched here: their recurrent state does not
* scale with max_t and survives on its own. */
if(m->Lc && max_t<=m->max_t) return;
float **oldL=m->Lc, **oldR=m->Rc;
int keep=(m->Lc && m->kvp.len>0 && m->kvp.len<=max_t) ? m->kvp.len : 0;
m->max_t=max_t;
m->Lc=calloc(c->n_layers,sizeof(float*));
m->Rc=calloc(c->n_layers,sizeof(float*));
for(int i=0;i<c->n_layers;i++) if(!m->L[i].kda){
m->Lc[i]=falloc((int64_t)max_t*c->kv_lora);
m->Rc[i]=falloc((int64_t)max_t*c->qk_rope);
if(keep){
memcpy(m->Lc[i], oldL[i], (size_t)keep*c->kv_lora*sizeof(float));
memcpy(m->Rc[i], oldR[i], (size_t)keep*c->qk_rope*sizeof(float));
}
}
if(oldL) for(int i=0;i<c->n_layers;i++){ free(oldL[i]); free(oldR[i]); }
free(oldL); free(oldR);
/* the record describes those same positions, so it survives with them */
if(!kv_prefix_grow(&m->kvp,max_t,keep)) kv_prefix_clear(&m->kvp);
/* DSA indexer cache — only for full layers (idx_type=1) */
for(int i=0;i<c->n_layers;i++){
Mla *a=&m->L[i].m;
if(c->index_hd > 0 && c->idx_type[i] && k3_dsa_indexer_on()){
a->Ic = falloc((int64_t)max_t * c->index_hd);
}
}
}
static void model_init(Model *m, const char *snap, int n_layers_env){
model_init_range(m,snap,0,n_layers_env,1,1,1,0,0,0);
}
typedef struct { float p; int id; } SampleProb;
static int sample_prob_desc(const void *a,const void *b){
float d=((const SampleProb*)b)->p-((const SampleProb*)a)->p;
return d>0?1:d<0?-1:0;
}
static int sample_tok(const float *lo, int V, float temp, float top_p){
if(temp<=0.f){ int b=0; for(int i=1;i<V;i++) if(lo[i]>lo[b]) b=i; return b; }
SampleProb *rank=malloc((size_t)V*sizeof(SampleProb)); float mx=lo[0];
if(!rank){ fprintf(stderr,"OOM sampling\n"); exit(1); }
for(int i=1;i<V;i++) if(lo[i]>mx) mx=lo[i];
double sum=0;
for(int i=0;i<V;i++){ float p=expf((lo[i]-mx)/temp); sum+=p; rank[i]=(SampleProb){p,i}; }
qsort(rank,(size_t)V,sizeof(SampleProb),sample_prob_desc);
double cut=(top_p>0.f&&top_p<1.f)?top_p*sum:sum, kept=0; int n=0;
while(n<V&&kept<cut) kept+=rank[n++].p;
double r=((double)rand()/RAND_MAX)*kept, acc=0; int pick=rank[0].id;
for(int i=0;i<n;i++){ acc+=rank[i].p; if(acc>=r){ pick=rank[i].id; break; } }
free(rank); return pick;
}
/* ---------- K3 XTML chat format (faithful to the shipped encoding_k3.py) --
* Only <|open|>, <|close|>, <|sep|>, <|end_of_msg|> are special TOKENS; tag
* names and attributes are ordinary text, encoded as the same standalone
* segments as the reference (segment boundaries are token boundaries). A
* turn renders as
* <|open|>message role="user"<|sep|>TEXT<|close|>message<|sep|><|end_of_msg|>
* and the generation prompt opens the assistant message plus its structural
* thinking channel:
* <|open|>message role="assistant"<|sep|><|open|>think<|sep|>
* (K3_THINK=0 opens <response> directly = non-thinking mode). The model then
* closes think, opens response, and finishes with <|end_of_msg|> (the eos). */
typedef struct { Tok *T; int *ids; int n, cap;
int sp_open, sp_close, sp_sep, sp_eom;
int cont; } ChatB; /* cont: the final turn was left open (continuation) */
static void cb_special(ChatB *b, int id){
if(b->n>=b->cap){ fprintf(stderr,"chat prompt too long\n"); exit(1); }
b->ids[b->n++]=id;
}
static void cb_text(ChatB *b, const char *s){
if(!*s) return;
b->n+=tok_encode(b->T,s,(int)strlen(s),b->ids+b->n,b->cap-b->n);
}
static void cb_open(ChatB *b, const char *tag, const char *role){
cb_special(b,b->sp_open); cb_text(b,tag);
if(role){ cb_text(b," role"); cb_text(b,"=\""); cb_text(b,role); cb_text(b,"\""); }
cb_special(b,b->sp_sep);
}
static void cb_close(ChatB *b, const char *tag){
cb_special(b,b->sp_close); cb_text(b,tag); cb_special(b,b->sp_sep);
}
static int chat_special(Tok *T, const char *s){
int l=(int)strlen(s);
for(int i=0;i<T->nsp;i++)
if(T->sp[i].len==l && !memcmp(T->sp[i].str,s,l)) return T->sp[i].id;
return -1;
}
/* returns prompt length; sp[4] = {open, close, sep, end_of_msg} ids */
static int chat_build(Tok *T, const char *sys, const char *user, int thinking,
int *ids, int cap, int *sp){
ChatB b={T,ids,0,cap,
chat_special(T,"<|open|>"), chat_special(T,"<|close|>"),
chat_special(T,"<|sep|>"), chat_special(T,"<|end_of_msg|>"), 0};
if(b.sp_open<0||b.sp_close<0||b.sp_sep<0||b.sp_eom<0){
fprintf(stderr,"chat: XTML special tokens not in tokenizer.json\n"); exit(1); }
sp[0]=b.sp_open; sp[1]=b.sp_close; sp[2]=b.sp_sep; sp[3]=b.sp_eom;
if(sys&&*sys){
cb_open(&b,"message","system"); cb_text(&b,sys);
cb_close(&b,"message"); cb_special(&b,b.sp_eom);
}
cb_open(&b,"message","user"); cb_text(&b,user);
cb_close(&b,"message"); cb_special(&b,b.sp_eom);
cb_open(&b,"message","assistant");
cb_open(&b,thinking?"think":"response",NULL);
return b.n;
}
static void chat_message(ChatB *b, const char *role, const char *text, int assistant){
cb_open(b,"message",role);
if(assistant) cb_open(b,"response",NULL);
cb_text(b,text);
if(assistant) cb_close(b,"response");
cb_close(b,"message");
cb_special(b,b->sp_eom);
}
static void chat_assistant(ChatB *b, const char *reasoning, const char *text){
cb_open(b,"message","assistant");
cb_open(b,"think",NULL); cb_text(b,reasoning); cb_close(b,"think");
cb_open(b,"response",NULL); cb_text(b,text); cb_close(b,"response");
cb_close(b,"message"); cb_special(b,b->sp_eom);
}
/* ---- tool-call XTML (#1143) -----------------------------------------------
* Attribute segmentation mirrors the reference renderer (encoding_k3.py):
* " key", "=\"", the escaped value, and "\"" are each their OWN text segment.
* Segment boundaries are token boundaries under K3's rank-BPE, so the split
* is part of the format, not a style choice. Values escape & -> & and
* " -> ", exactly the reference's _escape_attr_value. */
static void cb_attr(ChatB *b, const char *key, const char *val){
char kb[80]; snprintf(kb,sizeof kb," %s",key); cb_text(b,kb);
cb_text(b,"=\"");
size_t n=strlen(val), extra=0;
for(size_t i=0;i<n;i++) extra += val[i]=='&'?4u : val[i]=='"'?5u : 0u;
char *esc=malloc(n+extra+1);
if(!esc){ fprintf(stderr,"OOM chat attr\n"); exit(1); }
char *w=esc;
for(size_t i=0;i<n;i++){
if(val[i]=='&'){ memcpy(w,"&",5); w+=5; }
else if(val[i]=='"'){ memcpy(w,""",6); w+=6; }
else *w++=val[i];
}
*w=0; cb_text(b,esc); free(esc);
cb_text(b,"\"");
}
/* open tag with caller-supplied attributes: cb_open_begin, cb_attr..., cb_end */
static void cb_open_begin(ChatB *b, const char *tag){ cb_special(b,b->sp_open); cb_text(b,tag); }
static void cb_end(ChatB *b){ cb_special(b,b->sp_sep); }
/* Is a generated XTML tag part of the tool-call structure? Open tags carry
* attributes ("call tool=\"x\" index=\"1\""), close tags are the bare name. */
static int k3_tool_tag(const char *t){
return (!strncmp(t,"tools",5) &&(t[5]==0||t[5]==' ')) ||
(!strncmp(t,"call",4) &&(t[4]==0||t[4]==' ')) ||
(!strncmp(t,"argument",8)&&(t[8]==0||t[8]==' ')) ||
(!strncmp(t,"json",4) &&(t[4]==0||t[4]==' '));
}
/* Internal gateway payload. Length framing keeps arbitrary UTF-8/newlines in
* message content while preserving the segment boundaries required by K3's
* rank-BPE chat template:
* K3CHAT1\n
* M <role> <utf8-bytes>\n<content> ...
* G <thinking>\n
*/
static int chat_build_wire(Tok *T, const char *wire, int nwire, int *thinking,
int *ids, int cap, int *sp){
ChatB b={T,ids,0,cap,
chat_special(T,"<|open|>"), chat_special(T,"<|close|>"),
chat_special(T,"<|sep|>"), chat_special(T,"<|end_of_msg|>"), 0};
/* -2, not -1: the caller must be able to tell a bad payload from a snapshot
* whose tokenizer has no XTML tokens. Serve reported both as "invalid K3
* chat payload", which sent at least one user hunting through a request
* body that was perfectly well formed. The CLI path has always named the
* real cause; serve now does too. */
if(b.sp_open<0||b.sp_close<0||b.sp_sep<0||b.sp_eom<0) return -2;
sp[0]=b.sp_open; sp[1]=b.sp_close; sp[2]=b.sp_sep; sp[3]=b.sp_eom;
const char *p=wire, *end=wire+nwire;
if(nwire<8||memcmp(p,"K3CHAT1\n",8)) return -1;
p+=8; *thinking=0;
while(p<end){
const char *nl=memchr(p,'\n',(size_t)(end-p));
if(!nl) return -1;
if(b.cont && *p!='G') return -1; /* the open final turn must be last; only G may follow */
if(*p=='G'){
int v=0;
if(sscanf(p,"G %d",&v)!=1) return -1;
*thinking=!!v; p=nl+1; break;
}
if(*p=='A'){
int nr=-1, nt=-1;
if(sscanf(p,"A %d %d",&nr,&nt)!=2||nr<0||nt<0||nl+1+nr+nt>end) return -1;
char *reason=malloc((size_t)nr+1), *text=malloc((size_t)nt+1);
if(!reason||!text){ fprintf(stderr,"OOM chat assistant\n"); exit(1); }
memcpy(reason,nl+1,(size_t)nr); reason[nr]=0;
memcpy(text,nl+1+nr,(size_t)nt); text[nt]=0;
chat_assistant(&b,reason,text);
free(reason); free(text); p=nl+1+nr+nt; continue;
}
if(*p=='C'){ /* continuation: the FINAL assistant turn, left OPEN.
* C <reason-len> <text-len>\n<reason><text> -- the same
* body as A, but rendered as the trailing open turn (no
* <|close|>, no <|end_of_msg|>, and the caller appends no
* fresh generation cue). An old engine has no 'C' record:
* it falls to the 'M' branch below, fails to parse, and
* rejects the payload -- fail-closed, never miswired. */
int nr=-1, nt=-1;
if(sscanf(p,"C %d %d",&nr,&nt)!=2||nr<0||nt<0||nl+1+nr+nt>end) return -1;
char *reason=malloc((size_t)nr+1), *text=malloc((size_t)nt+1);
if(!reason||!text){ fprintf(stderr,"OOM chat continuation\n"); exit(1); }
memcpy(reason,nl+1,(size_t)nr); reason[nr]=0;
memcpy(text,nl+1+nr,(size_t)nt); text[nt]=0;
cb_open(&b,"message","assistant");
if(nr){ cb_open(&b,"think",NULL); cb_text(&b,reason); cb_close(&b,"think"); }
cb_open(&b,"response",NULL); cb_text(&b,text); /* OPEN: no close, no eom */
b.cont=1;
free(reason); free(text); p=nl+1+nr+nt; continue;
}
if(*p=='Y'){ /* typed system message (#1143): tool-declare / tool-choice */
int ntp=-1, nb=-1;
if(sscanf(p,"Y %d %d",&ntp,&nb)!=2||ntp<1||ntp>64||nb<0||nl+1+ntp+nb>end) return -1;
char *typ=malloc((size_t)ntp+1), *body=malloc((size_t)nb+1);
if(!typ||!body){ fprintf(stderr,"OOM chat typed-system\n"); exit(1); }
memcpy(typ,nl+1,(size_t)ntp); typ[ntp]=0;
memcpy(body,nl+1+ntp,(size_t)nb); body[nb]=0;
cb_open_begin(&b,"message"); cb_attr(&b,"role","system"); cb_attr(&b,"type",typ); cb_end(&b);
cb_text(&b,body);
cb_close(&b,"message"); cb_special(&b,b.sp_eom);
free(typ); free(body); p=nl+1+ntp+nb; continue;
}
if(*p=='O'){ /* tool result (#1143): O <index> <name-len> <content-len> */
int idx=-1, nn=-1, nb=-1;
if(sscanf(p,"O %d %d %d",&idx,&nn,&nb)!=3||idx<1||nn<1||nn>256||nb<0||nl+1+nn+nb>end) return -1;
char *name=malloc((size_t)nn+1), *body=malloc((size_t)nb+1);
if(!name||!body){ fprintf(stderr,"OOM chat tool-result\n"); exit(1); }
memcpy(name,nl+1,(size_t)nn); name[nn]=0;
memcpy(body,nl+1+nn,(size_t)nb); body[nb]=0;
char ib[16]; snprintf(ib,sizeof ib,"%d",idx);
cb_open_begin(&b,"message"); cb_attr(&b,"role","tool");
cb_attr(&b,"tool",name); cb_attr(&b,"index",ib); cb_end(&b);
cb_text(&b,body);
cb_close(&b,"message"); cb_special(&b,b.sp_eom);
free(name); free(body); p=nl+1+nn+nb; continue;
}
if(*p=='B'){ /* assistant WITH tool calls (#1143):
* B <think> <nr> <nt> <ncalls>\n<reason><text>
* then ncalls of F <name-len> <nargs>\n<name>
* (+ nargs of V <key-len> <type-len> <val-len>\n<key><type><val>)
* or J <name-len> <json-len>\n<name><json> */
int th=-1, nr=-1, nt=-1, nc=-1;
if(sscanf(p,"B %d %d %d %d",&th,&nr,&nt,&nc)!=4||th<0||th>1||nr<0||nt<0||
nc<1||nc>64||nl+1+nr+nt>end) return -1;
char *reason=malloc((size_t)nr+1), *text=malloc((size_t)nt+1);
if(!reason||!text){ fprintf(stderr,"OOM chat assistant-tools\n"); exit(1); }
memcpy(reason,nl+1,(size_t)nr); reason[nr]=0;
memcpy(text,nl+1+nr,(size_t)nt); text[nt]=0;
cb_open(&b,"message","assistant");
if(th){ cb_open(&b,"think",NULL); cb_text(&b,reason); cb_close(&b,"think"); }
cb_open(&b,"response",NULL); cb_text(&b,text); cb_close(&b,"response");
free(reason); free(text);
cb_open(&b,"tools",NULL);
p=nl+1+nr+nt;
for(int ci=0;ci<nc;ci++){
nl=memchr(p,'\n',(size_t)(end-p)); if(!nl) return -1;
char ib[16]; snprintf(ib,sizeof ib,"%d",ci+1);
if(*p=='J'){
int nn=-1, nj=-1;
if(sscanf(p,"J %d %d",&nn,&nj)!=2||nn<1||nn>256||nj<0||nl+1+nn+nj>end) return -1;
char *name=malloc((size_t)nn+1), *js=malloc((size_t)nj+1);
if(!name||!js){ fprintf(stderr,"OOM chat tool-call\n"); exit(1); }
memcpy(name,nl+1,(size_t)nn); name[nn]=0;
memcpy(js,nl+1+nn,(size_t)nj); js[nj]=0;
cb_open_begin(&b,"call"); cb_attr(&b,"tool",name); cb_attr(&b,"index",ib); cb_end(&b);
cb_open_begin(&b,"json"); cb_attr(&b,"type","object"); cb_end(&b);
cb_text(&b,js); cb_close(&b,"json"); cb_close(&b,"call");
free(name); free(js); p=nl+1+nn+nj;
} else if(*p=='F'){
int nn=-1, na=-1;
if(sscanf(p,"F %d %d",&nn,&na)!=2||nn<1||nn>256||na<0||na>64||nl+1+nn>end) return -1;
char *name=malloc((size_t)nn+1);
if(!name){ fprintf(stderr,"OOM chat tool-call\n"); exit(1); }
memcpy(name,nl+1,(size_t)nn); name[nn]=0;
cb_open_begin(&b,"call"); cb_attr(&b,"tool",name); cb_attr(&b,"index",ib); cb_end(&b);
free(name); p=nl+1+nn;
for(int ai=0;ai<na;ai++){
nl=memchr(p,'\n',(size_t)(end-p)); if(!nl) return -1;
int nk=-1, ntp=-1, nv=-1;
if(sscanf(p,"V %d %d %d",&nk,&ntp,&nv)!=3||nk<1||nk>256||ntp<1||ntp>16||
nv<0||nl+1+nk+ntp+nv>end) return -1;
char *key=malloc((size_t)nk+1), *typ=malloc((size_t)ntp+1), *val=malloc((size_t)nv+1);
if(!key||!typ||!val){ fprintf(stderr,"OOM chat tool-arg\n"); exit(1); }
memcpy(key,nl+1,(size_t)nk); key[nk]=0;
memcpy(typ,nl+1+nk,(size_t)ntp); typ[ntp]=0;
memcpy(val,nl+1+nk+ntp,(size_t)nv); val[nv]=0;
cb_open_begin(&b,"argument"); cb_attr(&b,"key",key); cb_attr(&b,"type",typ); cb_end(&b);
cb_text(&b,val); cb_close(&b,"argument");
free(key); free(typ); free(val); p=nl+1+nk+ntp+nv;
}
cb_close(&b,"call");
} else return -1;
}
cb_close(&b,"tools");
cb_close(&b,"message"); cb_special(&b,b.sp_eom);
continue;
}
char role[16]; int nb=-1;
if(sscanf(p,"M %15s %d",role,&nb)!=2||nb<0||nl+1+nb>end) return -1;
char *text=malloc((size_t)nb+1);
if(!text){ fprintf(stderr,"OOM chat message\n"); exit(1); }
memcpy(text,nl+1,(size_t)nb); text[nb]=0;
const char *r=!strcmp(role,"developer")?"system":role;
if(strcmp(r,"system")&&strcmp(r,"user")&&strcmp(r,"assistant")){ free(text); return -1; }
chat_message(&b,r,text,!strcmp(r,"assistant"));
free(text); p=nl+1+nb;
}
if(!b.cont){ /* a continuation already emitted the open final turn */
cb_open(&b,"message","assistant");
cb_open(&b,*thinking?"think":"response",NULL);
}
return b.n;
}
/* ---------- serve mode: shared openai_server.py protocol ---------- */
typedef struct {
char id[64];
int max_tok;
float temp, top_p;
char *payload;
int plen;
int logprobs, pin; /* SUBMIT logprobs=k / pin=1 */
} ServeReq;
static const ColiServeWireProfile kimi_wire={
.max_header_bytes=511,
.max_payload_bytes=1u<<24,
.max_tokens=1<<20,
.require_exact_lf=1,
.require_finite_sampling=0,
};
static void model_state_reset(Model *m){
Cfg *c=&m->c;
kv_prefix_clear(&m->kvp); /* the record describes the state we are dropping */
#ifdef COLI_VULKAN
k3c_host_wrote(m,1); /* the dense chain's copy is zeros too, its MLA rows gone */
#endif
for(int i=0;i<c->n_layers;i++){
if(m->L[i].kda){
memset(m->kstate[i],0,(size_t)c->kda_heads*c->kda_hd*c->kda_hd*sizeof(float));
memset(m->cwq[i],0,(size_t)c->kda_proj*c->conv_k*sizeof(float));
memset(m->cwk[i],0,(size_t)c->kda_proj*c->conv_k*sizeof(float));
memset(m->cwv[i],0,(size_t)c->kda_proj*c->conv_k*sizeof(float));
}
}
#ifdef COLI_METAL
if(m->Lc && m->max_t>0 && g_k3_metal){
/* KV cache is CPU-managed (host memory) even under Metal — clear in place and keep the
* buffers for reuse. coli_metal_kv_clear() wrote to an unsynced GPU copy, so the host
* cache stayed stale across turns; memset zeroes the memory attention actually reads. */
for(int i=0;i<c->n_layers;i++) if(!m->L[i].kda){
if(m->Lc[i]) memset(m->Lc[i],0,(size_t)m->max_t*c->kv_lora*sizeof(float));
if(m->Rc[i]) memset(m->Rc[i],0,(size_t)m->max_t*c->qk_rope*sizeof(float));
}
for(int i=0;i<c->n_layers;i++){
Mla *a=&m->L[i].m;
if(c->index_hd > 0 && c->idx_type[i] && a->Ic)
memset(a->Ic,0,(size_t)m->max_t*c->index_hd*sizeof(float));
}
} else
#endif
{
for(int i=0;i<c->n_layers;i++){
if(m->Lc && m->Lc[i]) free(m->Lc[i]);
if(m->Rc && m->Rc[i]) free(m->Rc[i]);
Mla *a=&m->L[i].m;
if(c->index_hd > 0 && c->idx_type[i] && a->Ic) free(a->Ic);
}
if(m->Lc) free(m->Lc);
if(m->Rc) free(m->Rc);
m->Lc=NULL; m->Rc=NULL; m->max_t=0;
}
}
/* ---------- recurrent-state checkpoints (OPT-IN, COLI_K3_CKPT=N) ----------
*
* kv_prefix reuse is all-or-nothing because the 69 KDA layers carry a single
* RECURRENT state that exists only at "now": a prompt that diverges anywhere
* before the fed length cannot be resumed, so today it resets everything and
* re-prefills from token 0 — on a streaming engine that is minutes of disk
* reads for an agentic edit that changed one tool result near the end.
*
* A checkpoint is a photograph of that recurrent state (kstate + the three
* conv windows of every KDA layer) taken at a turn boundary, together with
* its OWN copy of the token ids that produced it. On a divergent prompt the
* deepest photo still valid for the shared prefix is restored and only the
* tail is re-prefilled.
*
* Validity needs BOTH checks, not one: the photo's ids must prefix-match the
* new prompt (KDA side), and the restore position must not exceed the live
* common prefix of the CURRENT record (the MLA Lc/Rc rows are per-position
* and only valid while the record still describes them — after a reset the
* record is empty and no restore can happen, which is correct because
* model_state_reset also freed those rows).
*
* With COLI_K3_CKPT unset or 0 (the default), none of this code runs and the
* engine is byte-identical to before the feature existed. */
typedef struct {
int pos; /* state covers ids[0..pos-1] */
int *fed; /* own copy of those ids */
float *blob; /* packed kstate + cwq/cwk/cwv of every KDA layer */
float *logit; /* logit dell'ultima posizione coperta: predice il primo
* token fresco. Allocato solo per le foto chieste con
* SUBMIT pin=1, che sono le uniche a cui serve. */
int pinned; /* foto chiesta con pin=1: una foto ordinaria non la
* sfratta. Senza questo, con un solo slot la foto di
* fine risposta cancella quella del prompt e ogni
* opzione del menu rifa tutto da capo -- misurato. */
uint64_t used; /* LRU stamp */
} K3Ckpt;
static const float *g_k3_restored_logit=NULL; /* logit della foto appena rimessa */
static K3Ckpt g_k3_ckpt[K3_CKPT_MAX];
static uint64_t g_k3_ckpt_clock=0;
static size_t k3_ckpt_blob_floats(const Model *m){
const Cfg *c=&m->c;
size_t per=(size_t)c->kda_heads*c->kda_hd*c->kda_hd
+3u*(size_t)c->kda_proj*c->conv_k, n=0;
for(int i=0;i<c->n_layers;i++) if(m->L[i].kda) n+=per;
return n;
}
static double k3_ckpt_reserve_gb(const Model *m, int slots){
if(slots<1) return 0.0;
/* Disk-parked photos keep ONE bounce buffer in RAM regardless of slot
* count -- that is the whole point of COLI_K3_CKPT_DIR on a low-RAM
* machine. RAM slots each hold a full blob. */
int resident_blobs = g_k3_ckpt_dir ? 1 : slots;
return (double)resident_blobs*(double)k3_ckpt_blob_floats(m)*sizeof(float)/1e9;
}
static void k3_ckpt_copy(float *blob, Model *m, int to_blob){
Cfg *c=&m->c; size_t o=0;
#ifdef COLI_VULKAN
if(to_blob) k3c_sync_host(m); /* the dense chain may hold the newest state */
#endif
size_t ks=(size_t)c->kda_heads*c->kda_hd*c->kda_hd;
size_t cw=(size_t)c->kda_proj*c->conv_k;
for(int i=0;i<c->n_layers;i++) if(m->L[i].kda){
float *live[4]={m->kstate[i],m->cwq[i],m->cwk[i],m->cwv[i]};
size_t len[4]={ks,cw,cw,cw};
for(int j=0;j<4;j++){
if(to_blob) memcpy(blob+o,live[j],len[j]*sizeof(float));
else memcpy(live[j],blob+o,len[j]*sizeof(float));
o+=len[j];
}
}
}
/* Disk-parked photos: same photos, same validity rules, but the blob lives
* in a file so the RAM cost is one bounce buffer instead of N slots. Slot
* metadata (pos, fed ids, LRU stamp) stays in memory: photos never survive
* the process, because after a restart the per-position MLA rows are gone
* and no restore could ever be valid. Stale files from a previous run are
* simply overwritten. Writes are atomic (tmp + rename); any I/O failure
* drops the photo, never the request. */
static float *g_k3_ckpt_iobuf=NULL;
#define K3_CKPT_FILE_MAGIC ((((uint64_t)0x504B4333)<<32)|1u) /* "3CKP" v1 */
static void k3_ckpt_file(char *out, size_t n, int slot){
snprintf(out,n,"%s/k3_ckpt_%d.bin",g_k3_ckpt_dir,slot);
}
static int k3_ckpt_iobuf_ready(const Model *m){
if(!g_k3_ckpt_iobuf)
g_k3_ckpt_iobuf=malloc(k3_ckpt_blob_floats(m)*sizeof(float));
return g_k3_ckpt_iobuf!=NULL;
}
static int k3_ckpt_disk_write(Model *m, int slot, int pos){
char path[512], tmp[524];
k3_ckpt_file(path,sizeof path,slot);
snprintf(tmp,sizeof tmp,"%s.tmp",path);
size_t floats=k3_ckpt_blob_floats(m);
if(!k3_ckpt_iobuf_ready(m)) return 0;
k3_ckpt_copy(g_k3_ckpt_iobuf,m,1);
FILE *f=fopen(tmp,"wb"); if(!f) return 0;
uint64_t header[3]={K3_CKPT_FILE_MAGIC,(uint64_t)pos,(uint64_t)floats};
int ok=fwrite(header,sizeof header,1,f)==1 &&
fwrite(g_k3_ckpt_iobuf,sizeof(float),floats,f)==floats;
ok=!fclose(f)&&ok;
if(ok){ remove(path); ok=!rename(tmp,path); }
if(!ok) remove(tmp);
return ok;
}
static int k3_ckpt_disk_read(Model *m, int slot, int pos){
char path[512];
k3_ckpt_file(path,sizeof path,slot);
size_t floats=k3_ckpt_blob_floats(m);
if(!k3_ckpt_iobuf_ready(m)) return 0;
FILE *f=fopen(path,"rb"); if(!f) return 0;
uint64_t header[3]={0,0,0};
int ok=fread(header,sizeof header,1,f)==1 &&
header[0]==K3_CKPT_FILE_MAGIC &&
header[1]==(uint64_t)pos && header[2]==(uint64_t)floats &&
fread(g_k3_ckpt_iobuf,sizeof(float),floats,f)==floats;
fclose(f);
if(ok) k3_ckpt_copy(g_k3_ckpt_iobuf,m,0);
return ok;
}
/* A slot holds a photo: in RAM mode the blob pointer says so, in disk mode
* the committed position does (the file is only trusted through it). */
static int k3_ckpt_slot_full(const K3Ckpt *k){
return g_k3_ckpt_dir ? k->pos>0 : k->blob!=NULL;
}
static void k3_ckpt_save(Model *m, const float *logit){
if(g_k3_ckpt_slots<1) return;
int pos=m->kvp.len;
if(pos<1||m->kvp.tainted||!m->kvp.fed) return;
for(int s=0;s<g_k3_ckpt_slots;s++){ /* identical photo: refresh LRU */
K3Ckpt *k=&g_k3_ckpt[s];
if(k3_ckpt_slot_full(k)&&k->pos==pos&&
!memcmp(k->fed,m->kvp.fed,(size_t)pos*sizeof(int))){
if(logit){
if(!k->logit) k->logit=(float*)malloc((size_t)m->c.vocab*sizeof(float));
if(k->logit) memcpy(k->logit,logit,(size_t)m->c.vocab*sizeof(float));
k->pinned=1;
}
k->used=++g_k3_ckpt_clock; return; }
}
int victim=-1;
for(int s=0;s<g_k3_ckpt_slots;s++){
K3Ckpt *cand=&g_k3_ckpt[s];
if(!logit&&cand->pinned) continue; /* una foto ordinaria non la tocca */
if(!k3_ckpt_slot_full(cand)){ victim=s; break; }
if(victim<0||cand->used<g_k3_ckpt[victim].used) victim=s;
}
if(victim<0) return; /* tutte chieste: la foto ordinaria si salta */
K3Ckpt *k=&g_k3_ckpt[victim];
if(!g_k3_ckpt_dir&&!k->blob){
k->blob=malloc(k3_ckpt_blob_floats(m)*sizeof(float));
if(!k->blob) return; /* optimisation, never a failure */
}
int *fed=realloc(k->fed,(size_t)pos*sizeof(int));
if(!fed) return; /* slot keeps its old photo */
k->fed=fed;
memcpy(k->fed,m->kvp.fed,(size_t)pos*sizeof(int));
if(g_k3_ckpt_dir){
if(!k3_ckpt_disk_write(m,victim,pos)){ k->pos=0; return; }
} else k3_ckpt_copy(k->blob,m,1);
if(logit){
if(!k->logit) k->logit=(float*)malloc((size_t)m->c.vocab*sizeof(float));
if(k->logit) memcpy(k->logit,logit,(size_t)m->c.vocab*sizeof(float));
} else { free(k->logit); k->logit=NULL; }
k->pinned = logit ? 1 : 0;
k->pos=pos; k->used=++g_k3_ckpt_clock;
if(getenv("K3_PREFIX_LOG"))
fprintf(stderr,"[K3-CKPT] saved pos=%d slot=%d\n",pos,victim);
}
/* I logit di una foto che copre ESATTAMENTE `pos` token di questo prompt,
* senza rimettere niente. Serve quando il riuso vivo ha gia portato lo stato
* dove serve: la foto non va ripristinata, ma i suoi logit restano l'unico
* predittore del primo token fresco, e senza quelli la prima posizione
* dell'opzione esce senza logprob. */
static const float *k3_ckpt_logit_at(const int *ids, int pos){
if(g_k3_ckpt_slots<1||pos<1) return NULL;
for(int s=0;s<g_k3_ckpt_slots;s++){
const K3Ckpt *k=&g_k3_ckpt[s];
if(!k3_ckpt_slot_full(k)||k->pos!=pos||!k->logit) continue;
if(memcmp(k->fed,ids,(size_t)pos*sizeof(int))) continue;
return k->logit;
}
return NULL;
}
/* Deepest valid photo for this prompt; restores it and returns the covered
* positions (the caller prefills only ids[pos..np)), or 0 when none fits. */
static int k3_ckpt_restore_best(Model *m, const int *ids, int np){
g_k3_restored_logit=NULL;
if(g_k3_ckpt_slots<1||m->kvp.tainted||!m->kvp.fed) return 0;
int live=0, lim=m->kvp.len<np?m->kvp.len:np;
while(live<lim&&m->kvp.fed[live]==ids[live]) live++;
if(live>=np) live=np-1; /* at least one new token must be prefilled */
int best=-1;
for(int s=0;s<g_k3_ckpt_slots;s++){
K3Ckpt *k=&g_k3_ckpt[s];
if(!k3_ckpt_slot_full(k)||k->pos<1||k->pos>live) continue;
if(memcmp(k->fed,ids,(size_t)k->pos*sizeof(int))) continue;
if(best<0||k->pos>g_k3_ckpt[best].pos) best=s;
}
if(best<0){
/* Nessuna foto e servita a questo prompt: se ce n'era una chiesta, ha
* smesso di aiutare e si lascia andare, cosi la chat normale ritrova
* subito i suoi checkpoint. */
for(int s=0;s<g_k3_ckpt_slots;s++) g_k3_ckpt[s].pinned=0;
return 0;
}
K3Ckpt *k=&g_k3_ckpt[best];
if(g_k3_ckpt_dir){
if(!k3_ckpt_disk_read(m,best,k->pos)){ k->pos=0; return 0; }
} else k3_ckpt_copy(k->blob,m,0);
m->kvp.len=k->pos; /* the record truncates with the state it describes */
#ifdef COLI_VULKAN
k3c_host_wrote(m,0); /* the dense chain's copy goes up again */
#endif
g_k3_restored_logit=k->logit;
k->used=++g_k3_ckpt_clock;
if(getenv("K3_PREFIX_LOG"))
fprintf(stderr,"[K3-CKPT] restored pos=%d of %d prompt tokens "
"(recompute %d)\n",k->pos,np,np-k->pos);
return k->pos;
}
/* Decide reuse before changing the state it describes. A miss discards the
* old recurrent/KV state first and allocates a fresh target; a hit grows the
* existing target while preserving its prefix. The old order allocated first,
* then reset on the inevitable first-request miss, leaving Lc/Rc NULL when
* MLA prefill immediately indexed them (#855). */
static int prepare_request_state(Model *m, const int *ids, int np, int max_t){
int reuse=kv_prefix_reuse(&m->kvp,ids,np);
/* Divergent prompt: before surrendering to a full reset, try the
* checkpoint pool (a no-op unless COLI_K3_CKPT enabled it). */
if(!reuse) reuse=k3_ckpt_restore_best(m,ids,np);
if(!reuse) model_state_reset(m);
kv_alloc(m,max_t);
return reuse;
}
static int serve_stdin_readable(void){
/* Windows non ha fd_set/select in questa forma: la build falliva del tutto.
* La versione portabile (con i fix #139/#195) vive in compat.h, incluso via st.h. */
return coli_stdin_readable();
}
static int serve_read_req(FILE *in, FILE *out, ServeReq *q, const char *active){
ColiServeCommand command;
ColiServeReadResult result=coli_serve_read_command(in,&kimi_wire,&command);
if(result==COLI_SERVE_READ_EOF) return -1;
if(result==COLI_SERVE_READ_BAD_FRAME) return -2;
if(result==COLI_SERVE_READ_NOMEM){
coli_serve_write_error(out,command.id,"out of memory"); return -2;
}
if(result==COLI_SERVE_READ_BAD_REQUEST&&
command.kind==COLI_SERVE_COMMAND_SUBMIT){
/* SEC (GHSA-gf38): max_tok needs an upper bound too. INT_MAX wrapped the
* signed np+max_tok context check below and made the kv_alloc size
* negative, so kv_alloc's early-return kept the previous request's small
* KV buffers and the generation loop wrote past them (heap OOB write). */
coli_serve_write_error(out,command.id,"bad submit header"); return -2;
}
if(result!=COLI_SERVE_READ_OK) return 0;
if(command.kind==COLI_SERVE_COMMAND_STOP||
command.kind==COLI_SERVE_COMMAND_CANCEL){
int matched=active&&!strcmp(active,command.id);
coli_serve_command_dispose(&command); return matched;
}
if(command.kind!=COLI_SERVE_COMMAND_SUBMIT){
coli_serve_command_dispose(&command); return 0;
}
snprintf(q->id,sizeof(q->id),"%s",command.id);
q->max_tok=command.max_tokens; q->temp=command.temperature; q->top_p=command.top_p;
q->logprobs=command.logprobs; q->pin=command.pin;
q->payload=(char*)coli_serve_command_take_payload(&command);
q->plen=(int)command.payload_bytes;
coli_serve_command_dispose(&command);
return 2;
}
/* Drain only commands that can arrive while one request owns the engine. A
* second SUBMIT is refused without stealing the active request's state. */
typedef struct { const char *active; int fatal; } K3ServePoll;
static int k3_serve_poll_cancel(void *context){
K3ServePoll *poll=context;
int cancelled=0;
while(serve_stdin_readable()){
ServeReq queued={0}; int r=serve_read_req(stdin,stdout,&queued,poll->active);
if(r<0){ if(r==-2) poll->fatal=1; return 1; }
if(r==1) cancelled=1;
if(r==2){
coli_serve_write_error(stdout,queued.id,"engine busy");
free(queued.payload);
}
}
return cancelled;
}
/* A cancelled prefill changed recurrent/KV state without publishing a model
* token. Drop that partial state so a later request cannot reuse it. */
static void k3_cancel_unpublished_state(Model *m){
model_state_reset(m);
}
static void serve_data(const char *id, const char *p, int n){
if(n<=0) return;
coli_serve_write_data(stdout,id,p,(size_t)n);
}
/* Coda del canale logprobs sul frame DATA. La riempie il ciclo di decodifica
* per il token appena campionato; resta vuota per chi non ha chiesto nulla,
* e allora il frame e quello di sempre, byte per byte. */
static char g_lp_tail[1024];
static void serve_data_maybe_lp(const char *id, const char *p, int n){
if(n<=0) return;
if(g_lp_tail[0]) coli_serve_write_data_lp(stdout,id,p,(size_t)n,g_lp_tail);
else coli_serve_write_data(stdout,id,p,(size_t)n);
}
static void serve_tool(const char *id, const char *p, int n){
if(n<0) return;
coli_serve_write_tool(stdout,id,p,(size_t)n);
}
/* ---------- dashboard protocol: EMAP / HITS ----------
* Same stdout lines colibri.c emits for the web dashboard's Brain tab; the
* gateway parses them, nothing else does. Rows are the sparse layers, columns
* the experts. EMAP: one byte per expert as two hex digits, tier<<6 | heat
* (tier 1 = resident in the layer cache, 2 = on the Vulkan device, heat 0:
* this engine keeps no usage counter). Printed once after READY and again after every turn, because
* the cache fills as the turn runs. HITS: one bit per expert routed in the
* turn, packed 8 per hex pair, printed after DONE next to PROF. */
static void serve_emap(Model *m){
Cfg *c=&m->c; int E=c->n_experts, rows=0;
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) rows++;
char *hex=malloc((size_t)rows*E*2+1); int w=0;
for(int i=0;i<c->n_layers;i++){
if(!m->L[i].sparse) continue;
for(int e=0;e<E;e++){
int b=(vkt_resident(i,e)?2:slot_indexed(m,i,e)?1:0)<<6;
hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15];
}
}
hex[w]=0;
printf("EMAP %d %d %s\n",rows,E,hex); fflush(stdout); free(hex);
}
static void serve_hits(Model *m){
Cfg *c=&m->c; int E=c->n_experts, rows=0;
if(!m->ehit) ehit_mark(m,-1,-1); /* a turn that routed nothing still reports a bitmap: all zero */
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) rows++;
int nb=(rows*E+7)/8; uint8_t *bm=calloc((size_t)nb,1); int bit=0;
for(int i=0;i<c->n_layers;i++){
if(!m->L[i].sparse) continue;
for(int e=0;e<E;e++,bit++)
if(m->ehit[i][e]){ bm[bit>>3]|=(uint8_t)(1<<(bit&7)); m->ehit[i][e]=0; }
}
char *hex=malloc((size_t)nb*2+1); int w=0;
for(int b=0;b<nb;b++){ hex[w++]="0123456789abcdef"[bm[b]>>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; }
hex[w]=0;
printf("HITS %d %d %s\n",rows,E,hex); fflush(stdout); free(hex); free(bm);
}
/* When a request's prefill began, and the counters then: DONE and PROF report its share. */
typedef struct { double t0, a0, e0, d0, h0; uint64_t hit0, miss0; } K3ReqClock;
/* What a request's decoding needs from its start: the chat framing it was given. */
typedef struct { int chat, thinking, sp[4]; } K3ReqChat;
/* A request's prompt into the state the Model holds: its tokens (plain or K3CHAT1),
* the budget, ACCEPT, the prefix reuse and the photos, the prefill and its
* read-out. 1 with the prompt's ids and the logits after it; 0 when the request ended
* here, its ERROR written; -1 when the input is gone. poll: read CANCEL during the
* prefill (a lone serve), NULL to leave every command for later (serve_mux). */
static int k3_serve_start(Model *m, Tok *T, ServeReq *q, int **ids_out, int *np_out, float **lo_out,
K3ReqClock *clk, K3ReqChat *chatinfo, K3ServePoll *poll){
int cap=65536, *ids=malloc((size_t)cap*sizeof(int)), np=0;
if(!ids){ coli_serve_write_error(stdout,q->id,"out of memory"); return 0; }
int sp[4]={-1,-1,-1,-1}, chat=0, thinking=0;
if(m->c.bos>=0) ids[np++]=m->c.bos;
if(q->plen>=8&&!memcmp(q->payload,"K3CHAT1\n",8)){
int n=chat_build_wire(T,q->payload,q->plen,&thinking,ids+np,cap-np,sp);
if(n==-2){
/* Not the request's fault: this snapshot's tokenizer.json has no
* <|open|>/<|close|>/<|sep|>/<|end_of_msg|>, so no chat turn can be
* built from it. Say that, and say where a usable one comes from. */
coli_serve_write_error(stdout,q->id,
"tokenizer.json has no XTML chat tokens "
"(<|open|> <|close|> <|sep|> <|end_of_msg|>); "
"regenerate it with tools/k3_tokenizer.py");
fprintf(stderr,"[K3] chat: XTML special tokens not in tokenizer.json — "
"regenerate with tools/k3_tokenizer.py\n");
free(ids); return 0; }
if(n<0){ coli_serve_write_error(stdout,q->id,"invalid K3 chat payload"); free(ids); return 0; }
np+=n; chat=1;
} else {
np+=tok_encode(T,q->payload,q->plen,ids+np,cap-np);
}
int max_ctx=getenv("K3_MAXT")?atoi(getenv("K3_MAXT")):8192;
int budget=coli_serve_budget(np,q->max_tok,max_ctx,q->logprobs>0);
if(budget<0){
if(np<1){
coli_serve_write_error(stdout,q->id,"EMPTY_PROMPT");
}else{
char message[160];
snprintf(message,sizeof(message),
"CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
np,q->max_tok,max_ctx);
coli_serve_write_error(stdout,q->id,message);
}
free(ids); return 0;
}
if(budget<q->max_tok){
fprintf(stderr,"[serve] max_tokens %d clamped to %d (context %d - prompt %d); "
"raise K3_MAXT for longer answers\n",
q->max_tok,budget,max_ctx,np);
q->max_tok=budget;
}
coli_serve_write_accept(stdout,q->id,np);
/* Declare the structured sideband before any generated DATA. Even an
* empty sideband is meaningful: it tells the gateway that literal K3
* marker lookalikes in DATA are content, not engine-proven structure. */
if(chat) serve_tool(q->id,NULL,0);
/* KV PREFIX REUSE (#639 for GLM; this engine re-prefilled every turn).
* A chat client resends the whole transcript each turn, so turn N used to
* re-process turns 1..N-1 from scratch — the cost of a message grew with
* the conversation, and every replayed position pulled its experts off
* disk again. When this prompt begins with the sequence the state already
* holds, that state IS the state at that position: keep it and prefill
* only the tail. The reuse decision must happen BEFORE a miss resets that
* state; allocation then either starts fresh or grows the preserved state.
* At least one new token is required, since the state cannot be rewound.
* Either the reused positions are token-identical or nothing is reused;
* the emitted tokens are unchanged in both cases. */
/* Una richiesta con pin=1 chiede la fotografia anche quando l'operatore non
* ha acceso COLI_K3_CKPT: senza almeno uno slot non c'e dove metterla e
* ogni opzione rifarebbe il prompt intero. Se la memoria non basta si
* resta senza: e un'ottimizzazione, mai il motivo di un errore. */
/* Gli scatti utili sono ANNIDATI: le istruzioni, condivise da mille
* richieste, e istruzioni+domanda, condivise dalle alternative di una
* sola. Con uno solo se ne perde sempre uno e la domanda viene riletta
* una volta per opzione. Questo motore il pool ce l'ha gia' (K3Ckpt tiene
* il piu' profondo valido); qui si apre solo con quanti slot, con lo
* stesso COLI_PIN_SLOTS degli altri. */
if(q->pin && g_k3_ckpt_slots<1){
int n=coli_pin_slots_wanted(); if(n<1) n=1;
if(n>K3_CKPT_MAX) n=K3_CKPT_MAX;
g_k3_ckpt_slots=n;
fprintf(stderr,"[PIN] %d scatti dello stato ricorrente accesi su richiesta (%.2f GB)\n",
n,k3_ckpt_reserve_gb(m,n));
}
int reuse=prepare_request_state(m,ids,np,np+q->max_tok+8);
/* Il riporto per il primo token fresco: i logit salvati con la foto che si
* e appena rimessa. Senza foto resta spento e la prima posizione esce con
* " nan 0", come deve. */
g_echo_k=q->logprobs; g_echo_id=q->id; g_echo_tok=T;
g_echo_carry_ok=0;
if(q->logprobs>0){
if(!g_echo_carry) g_echo_carry=(float*)malloc((size_t)m->c.vocab*sizeof(float));
const float *pred = g_k3_restored_logit;
if(!pred&&reuse>0) pred=k3_ckpt_logit_at(ids,reuse);
if(g_echo_carry&&pred&&reuse>0){
memcpy(g_echo_carry,pred,(size_t)m->c.vocab*sizeof(float));
g_echo_carry_ok=1;
}
}
if(getenv("K3_PREFIX_LOG")){
/* Report the decision either way, with the state behind a "no".
* "It did not get faster" is otherwise the same observation as
* "reuse is not wired up" — for a user as much as for a test. */
if(reuse)
fprintf(stderr,"[PREFIX] reusing %d of %d prompt tokens (%.0f%%)\n",
reuse,np,100.0*reuse/np);
else
fprintf(stderr,"[PREFIX] no reuse: held=%d cap=%d prompt=%d%s\n",
m->kvp.len,m->kvp.cap,np,
(m->kvp.len>0 && m->kvp.len<np) ? " (diverged)" : "");
fflush(stderr);
}
int chunk=getenv("K3_CHUNK")?atoi(getenv("K3_CHUNK")):32;
if(chunk<1) chunk=1; if(chunk>512) chunk=512;
#ifdef COLI_VULKAN
{ int r=k3c_prefill_rows(m); if(r>0 && !g_lfp) chunk=r; } /* the chain's chunk from the budget: big MoE steps */
#endif
clk->t0=now_s(); clk->a0=m->t_attn; clk->e0=m->t_moe; clk->d0=m->t_eload; clk->h0=m->t_head;
clk->hit0=m->hits; clk->miss0=m->miss;
float *lo=NULL;
/* `i` is the ABSOLUTE position: attention and the MLA Lc/Rc slots are
* position-indexed, so the loop starts at `reuse`, not at 0. */
int prefill_cancelled=0;
for(int i=reuse;i<np;i+=chunk){
int C=np-i<chunk?np-i:chunk;
free(lo); lo=step_chunk_ex(m,ids+i,i,C,poll?k3_serve_poll_cancel:NULL,poll,
&prefill_cancelled);
if(prefill_cancelled) break;
}
if(prefill_cancelled){
free(lo); k3_cancel_unpublished_state(m); free(ids);
if(!poll->fatal) coli_serve_write_error(stdout,q->id,"CANCELLED");
return poll->fatal?-1:0;
}
/* Turn boundary: the state now covers exactly the prompt. A photo here is
* what an agentic edit of the NEXT request restores from. */
k3_ckpt_save(m, q->pin ? lo : NULL);
if(q->pin) fprintf(stderr,"[PIN] stato fotografato a %d token\n",np);
g_echo_k=0; g_echo_id=NULL; /* la lettura riguarda il prefill, non la decodifica */
*ids_out=ids; *np_out=np; *lo_out=lo;
chatinfo->chat=chat; chatinfo->thinking=thinking; memcpy(chatinfo->sp,sp,sizeof sp);
return 1;
}
static int serve_one(Model *m, Tok *T, ServeReq *q){
int *ids=NULL, np=0; float *lo=NULL; K3ReqClock clk; K3ReqChat ci;
K3ServePoll poll={q->id,0};
int started=k3_serve_start(m,T,q,&ids,&np,&lo,&clk,&ci,&poll);
if(started<=0) return started;
int chat=ci.chat, thinking=ci.thinking, *sp=ci.sp;
double t0=clk.t0, a0=clk.a0, e0=clk.e0, d0=clk.d0, h0=clk.h0;
uint64_t hit0=clk.hit0, miss0=clk.miss0;
int gen=0, limited=1, cancelled=0, xsup=0, xopen=0, xtl=0, xtool=0;
char buf[512], xtag[320]; /* tool-call open tags carry attributes: call tool="..." index="..." (#1143) */
double tg=now_s();
int forwards=1; /* the prefill; decode steps are counted where they run */
for(int s=0;s<q->max_tok&&!cancelled;s++){
int tk=sample_tok(lo,m->c.vocab,q->temp,q->top_p);
g_lp_tail[0]=0;
if(q->logprobs>0) coli_logprob_tail(g_lp_tail,sizeof g_lp_tail,lo,m->c.vocab,tk,q->logprobs);
free(lo); lo=NULL;
int eos=0; for(int i=0;i<m->c.n_eos;i++) if(tk==m->c.eos[i]) eos=1;
int show=!eos;
if(chat&&sp[0]>=0){
if(tk==sp[0]||tk==sp[1]){
xsup=1; xopen=(tk==sp[0]); xtl=0; show=0;
} else if(tk==sp[2]){
if(xsup){
xsup=0; xtag[xtl]=0;
if(xopen&&!strcmp(xtag,"response")&&thinking)
serve_data(q->id,"</think>",8);
else if(k3_tool_tag(xtag)){
/* #1147: preserve engine-proven special-token structure on the
* TOOL sideband. Text that only resembles these markers remains
* DATA and cannot be promoted by the gateway. */
char lb[352];
int n=snprintf(lb,sizeof lb,"%s%s<|sep|>",xopen?"<|open|>":"<|close|>",xtag);
if(n>0&&n<(int)sizeof lb) serve_tool(q->id,lb,n);
if(!strcmp(xtag,"tools")) xtool=xopen;
}
}
show=0;
} else if(xsup){
int nb=tok_decode(T,&tk,1,buf,sizeof(buf)-1);
if(xtl+nb<(int)sizeof(xtag)){ memcpy(xtag+xtl,buf,(size_t)nb); xtl+=nb; }
show=0;
} else if(tk==sp[3]) show=0;
}
if(show){
int nb=tok_decode(T,&tk,1,buf,sizeof(buf)-1);
if(xtool) serve_tool(q->id,buf,nb);
else serve_data_maybe_lp(q->id,buf,nb);
}
if(!eos) gen++;
while(serve_stdin_readable()){
ServeReq queued={0};
int r=serve_read_req(stdin,stdout,&queued,q->id);
if(r==-2){ poll.fatal=1; cancelled=1; break; }
if(r<0){ cancelled=1; break; }
if(r==1) cancelled=1;
if(r==2){
coli_serve_write_error(stdout,queued.id,"engine busy"); free(queued.payload);
}
}
if(cancelled){ limited=0; break; }
if(eos){ limited=0; break; }
if(s+1<q->max_tok){ lo=step_chunk(m,&tk,np+s,1); forwards++; }
}
if(poll.fatal){ free(lo); free(ids); return -1; }
free(lo); free(ids);
/* End of the reply: the next turn's transcript extends THIS state, and an
* edited retry of this turn restores the prompt-boundary photo above. */
k3_ckpt_save(m, NULL);
double dt=now_s()-t0, decode=now_s()-tg;
uint64_t hits=m->hits-hit0, misses=m->miss-miss0, total=hits+misses;
ColiServeDone done={gen,decode>0?gen/decode:0.0,
total?100.0*hits/total:0.0,rss_gb(),np,limited};
char done_line[256];
int done_bytes=coli_serve_format_done(done_line,sizeof(done_line),q->id,&done);
if(done_bytes>0) fwrite(done_line,1,(size_t)done_bytes,stdout);
double moe=m->t_moe-e0, disk=m->t_eload-d0;
printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n",
dt,np,gen,disk,0.0,moe>disk?moe-disk:moe,m->t_attn-a0,m->t_head-h0,forwards);
fflush(stdout);
serve_hits(m);
#ifdef COLI_VULKAN
k3_vk_report(m,"turn");
#endif
return 0;
}
/* ---- several conversations at once (KV_SLOTS>1) -------------------------------
* The gateway's cache slots, each a conversation with a state of its own (K3Seq). A
* SUBMIT on a free slot starts its request at once through k3_serve_start, on that
* slot's state: its prefix reuse and photos work as a lone serve's (its prefill is
* not interrupted: the commands wait for it). Then every step picks the next token of
* each active request and runs one forward over a row of each (k3_step_rows): the
* matrices and the experts are read once for all of them. A request's frames, its
* chat markers and tool sideband included, are a lone request's; they interleave by
* id. As alone, STOP and CANCEL end a request with DONE. */
static K3Seq *g_k3_mux_seq; /* [slots]: the conversations the Model does not hold */
static int g_k3_mux_cur; /* the slot the Model holds, -1 when every one is parked */
typedef struct {
ServeReq q;
int active, cancel, limited, gen, s, forwards;
int *ids, np;
float *lo; /* the logits the next pick reads */
K3ReqClock clk;
K3ReqChat ci;
int xsup, xopen, xtl, xtool;
char xtag[320];
double tg; /* when its decoding began */
} K3MuxReq;
static void k3_mux_bind(Model *m, int slot){
if(g_k3_mux_cur==slot) return;
if(g_k3_mux_cur>=0) k3_seq_swap(m,&g_k3_mux_seq[g_k3_mux_cur]);
if(slot>=0) k3_seq_swap(m,&g_k3_mux_seq[slot]);
g_k3_mux_cur=slot;
}
/* A request's end, as serve_one ends one. */
static void k3_mux_finish(Model *m, K3MuxReq *r, int slot){
free(r->lo); r->lo=NULL; free(r->ids); r->ids=NULL; r->active=0;
if(r->cancel) r->limited=0;
/* the end of the reply: the next turn extends THIS state (a photo when they are on) */
k3_mux_bind(m,slot); k3_ckpt_save(m,NULL);
double dt=now_s()-r->clk.t0, decode=now_s()-r->tg;
uint64_t hits=m->hits-r->clk.hit0, misses=m->miss-r->clk.miss0, total=hits+misses;
ColiServeDone done={r->gen,decode>0?r->gen/decode:0.0,total?100.0*hits/total:0.0,rss_gb(),r->np,r->limited};
char done_line[256];
int done_bytes=coli_serve_format_done(done_line,sizeof(done_line),r->q.id,&done);
if(done_bytes>0) fwrite(done_line,1,(size_t)done_bytes,stdout);
double moe=m->t_moe-r->clk.e0, disk=m->t_eload-r->clk.d0;
printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n",
dt,r->np,r->gen,disk,0.0,moe>disk?moe-disk:moe,m->t_attn-r->clk.a0,m->t_head-r->clk.h0,r->forwards);
fflush(stdout);
serve_hits(m);
}
/* The next token of an active request, as serve_one's loop picks and sends it (the
* chat markers to their channels): 1 with the token when the request goes on, 0 when
* it ended. */
static int k3_mux_pick(Model *m, Tok *T, K3MuxReq *r, int slot, int *tk_out){
if(r->cancel||r->s>=r->q.max_tok){ k3_mux_finish(m,r,slot); return 0; }
int tk=sample_tok(r->lo,m->c.vocab,r->q.temp,r->q.top_p);
g_lp_tail[0]=0;
if(r->q.logprobs>0) coli_logprob_tail(g_lp_tail,sizeof g_lp_tail,r->lo,m->c.vocab,tk,r->q.logprobs);
free(r->lo); r->lo=NULL;
int eos=0; for(int i=0;i<m->c.n_eos;i++) if(tk==m->c.eos[i]) eos=1;
int show=!eos, *sp=r->ci.sp;
char buf[512];
if(r->ci.chat&&sp[0]>=0){
if(tk==sp[0]||tk==sp[1]){
r->xsup=1; r->xopen=(tk==sp[0]); r->xtl=0; show=0;
} else if(tk==sp[2]){
if(r->xsup){
r->xsup=0; r->xtag[r->xtl]=0;
if(r->xopen&&!strcmp(r->xtag,"response")&&r->ci.thinking)
serve_data(r->q.id,"</think>",8);
else if(k3_tool_tag(r->xtag)){
char lb[352];
int n=snprintf(lb,sizeof lb,"%s%s<|sep|>",r->xopen?"<|open|>":"<|close|>",r->xtag);
if(n>0&&n<(int)sizeof lb) serve_tool(r->q.id,lb,n);
if(!strcmp(r->xtag,"tools")) r->xtool=r->xopen;
}
}
show=0;
} else if(r->xsup){
int nb=tok_decode(T,&tk,1,buf,sizeof(buf)-1);
if(r->xtl+nb<(int)sizeof(r->xtag)){ memcpy(r->xtag+r->xtl,buf,(size_t)nb); r->xtl+=nb; }
show=0;
} else if(tk==sp[3]) show=0;
}
if(show){
int nb=tok_decode(T,&tk,1,buf,sizeof(buf)-1);
if(r->xtool) serve_tool(r->q.id,buf,nb);
else serve_data_maybe_lp(r->q.id,buf,nb);
}
if(!eos) r->gen++;
r->s++;
if(eos){ r->limited=0; k3_mux_finish(m,r,slot); return 0; }
if(r->s>=r->q.max_tok){ k3_mux_finish(m,r,slot); return 0; }
*tk_out=tk; return 1;
}
static void serve_mux(Model *m, Tok *T){
int n=g_k3_mux_slots, V=m->c.vocab, input_eof=0;
K3MuxReq *rq=calloc((size_t)n,sizeof *rq);
K3Row *rows=malloc((size_t)n*sizeof *rows);
int *tok=malloc((size_t)n*sizeof(int)), *who=malloc((size_t)n*sizeof(int));
if(!rq||!rows||!tok||!who){ fprintf(stderr,"[serve] out of memory\n"); exit(1); }
unsigned long long steps=0, nrows=0;
fprintf(stderr,"[kimi_k3] serving %d conversations at once (KV_SLOTS)\n",n);
for(;;){
int active=0; for(int i=0;i<n;i++) active+=rq[i].active;
/* idle: wait for a command; decoding: take one only when one is there */
if(!input_eof&&(!active||serve_stdin_readable())){
ColiServeCommand command;
ColiServeReadResult result=coli_serve_read_command(stdin,&kimi_wire,&command);
if(result==COLI_SERVE_READ_EOF||result==COLI_SERVE_READ_BAD_FRAME) input_eof=1;
else if(result==COLI_SERVE_READ_NOMEM){ coli_serve_write_error(stdout,command.id,"out of memory"); input_eof=1; }
else if(result==COLI_SERVE_READ_BAD_REQUEST){
if(command.kind==COLI_SERVE_COMMAND_SUBMIT) coli_serve_write_error(stdout,command.id,"bad submit header");
coli_serve_command_dispose(&command);
} else if(result==COLI_SERVE_READ_OK){
if(command.kind==COLI_SERVE_COMMAND_STOP||command.kind==COLI_SERVE_COMMAND_CANCEL){
for(int i=0;i<n;i++) if(rq[i].active&&!strcmp(rq[i].q.id,command.id)) rq[i].cancel=1;
} else if(command.kind==COLI_SERVE_COMMAND_SUBMIT){
if(command.slot<0||command.slot>=n) coli_serve_write_error(stdout,command.id,"invalid cache slot");
else if(rq[command.slot].active) coli_serve_write_error(stdout,command.id,"SLOT_BUSY");
else {
K3MuxReq *t=&rq[command.slot];
memset(t,0,sizeof *t);
snprintf(t->q.id,sizeof(t->q.id),"%s",command.id);
t->q.max_tok=command.max_tokens; t->q.temp=command.temperature; t->q.top_p=command.top_p;
t->q.logprobs=command.logprobs; t->q.pin=command.pin;
t->q.payload=(char*)coli_serve_command_take_payload(&command);
t->q.plen=(int)command.payload_bytes;
k3_mux_bind(m,command.slot);
int ok=k3_serve_start(m,T,&t->q,&t->ids,&t->np,&t->lo,&t->clk,&t->ci,NULL);
free(t->q.payload); t->q.payload=NULL;
if(ok>0){ t->active=1; t->limited=1; t->forwards=1; t->tg=now_s(); }
}
}
coli_serve_command_dispose(&command);
}
}
active=0; for(int i=0;i<n;i++) active+=rq[i].active;
if(!active){ if(input_eof) break; continue; }
int S=0, ended=0;
for(int i=0;i<n;i++) if(rq[i].active){
int tk;
if(!k3_mux_pick(m,T,&rq[i],i,&tk)){ ended=1; continue; }
rows[S]=(K3Row){&g_k3_mux_seq[i],rq[i].np+rq[i].s-1}; tok[S]=tk; who[S]=i; S++;
}
if(S){
k3_mux_bind(m,-1); /* every conversation parked: the rows read theirs */
float *lo=k3_step_rows(m,rows,tok,S);
steps++; nrows+=(unsigned long long)S;
for(int s=0;s<S;s++){
K3MuxReq *t=&rq[who[s]];
t->lo=falloc(V); memcpy(t->lo,lo+(int64_t)s*V,(size_t)V*sizeof(float));
t->forwards++;
}
free(lo);
}
if(ended){
#ifdef COLI_VULKAN
k3_vk_report(m,"turn");
#endif
serve_emap(m);
}
}
fprintf(stderr,"[kimi_k3] KV_SLOTS=%d: %llu decode steps, %llu rows (%.2f a step)\n",n,steps,nrows,
steps?(double)nrows/(double)steps:0.0);
k3_mux_bind(m,0);
free(rq); free(rows); free(tok); free(who);
}
static void serve_loop(Model *m, Tok *T){
/* PRIMA del sentinella: su Windows stdout in modalita' TEXT trasforma il \n
* finale in \r\n, il gateway non lo riconosce e resta in attesa per sempre
* (#748). Vive in compat.h perche' colibri.c ce l'ha da #195 e questo motore
* e' nato senza. */
coli_serve_stdio_init();
coli_serve_write_ready(stdout,rss_gb());
serve_emap(m); /* after READY and STAT: the boot reader discards what precedes them */
if(g_k3_mux_slots>1){ serve_mux(m,T); return; }
for(;;){
ServeReq q={0}; int r;
do r=serve_read_req(stdin,stdout,&q,NULL); while(r==0);
if(r<0) return;
if(r==2){ int fatal=serve_one(m,T,&q); free(q.payload); if(fatal<0) return; serve_emap(m); }
}
}
#ifndef KIMI_K3_NO_MAIN
int main(int argc, char **argv){
coli_omp_tune_threads("kimi_k3"); /* squadra sui core fisici, niente spin-wait: vedi omp_tune.h */
int serving=getenv("SERVE")&&getenv("SERVE")[0]=='1';
if(serving){ /* KV_SLOTS: the conversations a serve decodes at once */
const char *ks=getenv("KV_SLOTS");
if(ks&&*ks){
char *end=NULL; long v=strtol(ks,&end,10);
if(end==ks||*end||v<1||v>16){ fprintf(stderr,"KV_SLOTS must be between 1 and 16\n"); return 2; }
g_k3_mux_slots=(int)v;
}
}
/* Usage was printed only when there were NO arguments, so `--help` fell
* through as the model directory and the engine went looking for
* "--help/config.json". Nobody should have to read the source to find the
* argument order. Print it for the help flags too, and from every refusal
* below, so an error says what to do instead of only what went wrong. */
if(!serving && (argc<2 || !strcmp(argv[1],"--help") || !strcmp(argv[1],"-h")
|| !strcmp(argv[1],"help"))){
if(argc<2){ coli_print_launcher_help("Kimi K3", "./kimi_k3 <model directory> ...; --help lists the options"); return 1; }
k3_usage(argv[0]);
return argc<2 ? 1 : 0; /* asking for help is not a failure */
}
const char *snap=serving?getenv("SNAP"):argv[1], *prompt=NULL, *idstr=NULL, *sysmsg=NULL, *wirepath=NULL;
if(!snap||!*snap){ fprintf(stderr,"set SNAP=<Kimi K3 snapshot directory>\n"); return 1; }
int ngen=32, chat=0;
for(int i=serving?1:2;i<argc;i++){
if(!strcmp(argv[i],"--ngen")&&i+1<argc) ngen=coli_arg_int(argv[++i], "--ngen");
else if(!strcmp(argv[i],"--ids")&&i+1<argc) idstr=argv[++i];
else if(!strcmp(argv[i],"--chat")) chat=1;
else if(!strcmp(argv[i],"--system")&&i+1<argc) sysmsg=argv[++i];
else if(!strcmp(argv[i],"--wire-test")&&i+1<argc) wirepath=argv[++i];
else if(!prompt) prompt=argv[i];
}
if(wirepath){
char tp[2048]; snprintf(tp,sizeof(tp),"%s/tokenizer.json",snap);
Tok wt; tok_load(&wt,tp);
FILE *wf=fopen(wirepath,"rb"); if(!wf){ perror(wirepath); return 1; }
fseek(wf,0,SEEK_END); long wn=ftell(wf); fseek(wf,0,SEEK_SET);
if(wn<0||wn>(1<<24)){ fprintf(stderr,"wire payload too large\n"); fclose(wf); return 1; }
char *wire=malloc((size_t)wn+1); int *wid=malloc(65536*sizeof(int));
if(!wire||!wid){ fprintf(stderr,"OOM wire test\n"); return 1; }
if(fread(wire,1,(size_t)wn,wf)!=(size_t)wn){ fprintf(stderr,"short wire read\n"); return 1; }
fclose(wf); wire[wn]=0;
int thinking=0, wsp[4], n=chat_build_wire(&wt,wire,(int)wn,&thinking,wid,65536,wsp);
if(n<0){ fprintf(stderr,"invalid K3 chat wire payload\n"); return 1; }
for(int i=0;i<n;i++) printf("%s%d",i?" ":"",wid[i]);
printf("\n"); free(wire); free(wid); return 0;
}
float temp=getenv("COLI_TEMP")?(float)atof(getenv("COLI_TEMP")):0.f;
int nlayers=getenv("K3_LAYERS")?atoi(getenv("K3_LAYERS")):0;
Model m;
model_init(&m,snap,nlayers);
rt_init("kimi_k3",m.c.n_layers,m.c.n_experts); /* counters, identity, ROUTE_TRACE */
/* A layer with no counter row is a layer that cannot be credited: dense layers do not
* route, and K3 has no MTP row. Without this a history record naming one of them is
* silently absorbed and written back out. See docs/routing-telemetry.md. */
for(int i=0;i<m.c.n_layers;i++) if(!m.L[i].sparse) rt_drop_row(i);
rt_drop_row(m.c.n_layers);
/* LEARNED CACHE. Expert use accumulates in <snap>/.coli_usage across sessions
* and seeds the pins at startup, the same convention colibri.c and inkling.c
* already follow. COLI_USAGE overrides the location; USAGE_SAVE=0 makes the
* run read-only, for benchmark loops that would otherwise skew the profile
* they are measuring. */
{ const char *up=getenv("COLI_USAGE");
if(up&&*up) snprintf(g_k3_usage,sizeof(g_k3_usage),"%s",up);
else snprintf(g_k3_usage,sizeof(g_k3_usage),"%s/.coli_usage",snap);
int64_t h=rt_load(g_k3_usage);
if(h>0) fprintf(stderr,"[USAGE] expert history: %lld selections (%s)\n",
(long long)h,g_k3_usage);
/* #780: without a half-life the ranking freezes — after ~18M recorded
* selections one more turn moves it by 0.2% and the profile stops
* following the workload. */
if(getenv("COLI_USAGE_DECAY")) rt_decay();
pin_seed(&m,h); }
#ifdef COLI_VULKAN
k3_vk_tier_start(&m); /* COLI_VULKAN=1: hot routed experts on the device, from the history above */
#endif
if(getenv("K3_TRACE")){
m.trace=fopen(getenv("K3_TRACE"),"wb");
if(!m.trace){ perror(getenv("K3_TRACE")); return 1; }
}
if(getenv("K3_X0")){ /* injected input rows [T,hidden] f32, bypasses embed */
FILE *f=fopen(getenv("K3_X0"),"rb");
if(!f){ perror(getenv("K3_X0")); return 1; }
fseek(f,0,SEEK_END); long fn=ftell(f); fseek(f,0,SEEK_SET);
g_x0_n=(int)(fn/((long)m.c.hidden*4));
g_x0=falloc((int64_t)g_x0_n*m.c.hidden);
if(fread(g_x0,4,(size_t)g_x0_n*m.c.hidden,f)!=(size_t)g_x0_n*m.c.hidden){ fprintf(stderr,"K3_X0 short read\n"); return 1; }
fclose(f);
fprintf(stderr,"[K3] K3_X0: %d injected input rows\n",g_x0_n);
}
/* tokenize */
int ids[65536], np=0;
Tok T; int has_tok=0;
{ char tp[2048]; snprintf(tp,sizeof(tp),"%s/tokenizer.json",snap);
FILE *f=fopen(tp,"rb"); if(f){ fclose(f); tok_load(&T,tp); has_tok=1;
fprintf(stderr,"[K3] tokenizer.json loaded (family=%s)\n",T.kimi?"kimi":(T.o200k?"o200k":"cl100k")); } }
if(serving){
if(!has_tok){ fprintf(stderr,"serve mode needs tokenizer.json\n"); return 1; }
coli_rt_term_arm(); /* SIGTERM must reach the save below (#1629) */
if(g_k3_mux_slots>1){ /* slot 0 is the Model's own state */
g_k3_mux_seq=calloc((size_t)g_k3_mux_slots,sizeof *g_k3_mux_seq);
if(!g_k3_mux_seq){ fprintf(stderr,"[serve] out of memory for %d conversations\n",g_k3_mux_slots); return 1; }
for(int i=1;i<g_k3_mux_slots;i++) k3_seq_alloc(&m,&g_k3_mux_seq[i]);
g_k3_mux_cur=0;
}
serve_loop(&m,&T);
/* Questo ramo non salvava affatto la storia: gli altri motori lo
* fanno subito dopo il loop, qui mancava del tutto, quindi la
* modalita serve non ha mai contribuito alla cache appresa --
* nemmeno uscendo in modo pulito. */
if(g_k3_usage[0]) rt_save(g_k3_usage,0);
if(g_k3_val_fp){ fflush(g_k3_val_fp); fclose(g_k3_val_fp); g_k3_val_fp=NULL; }
if(g_k3_val_lfp){ fflush(g_k3_val_lfp); fclose(g_k3_val_lfp); g_k3_val_lfp=NULL; }
#ifdef COLI_METAL
if(g_k3_metal) coli_metal_shutdown();
#endif
return 0;
}
int sp[4]={-1,-1,-1,-1};
int think=getenv("K3_THINK")?atoi(getenv("K3_THINK")):1;
if(idstr){
const char *p=idstr;
while(*p&&np<65536){ while(*p==' '||*p==',')p++; if(!*p)break; ids[np++]=(int)strtol(p,(char**)&p,10); }
} else if(chat){
if(!has_tok){ fprintf(stderr,"--chat needs tokenizer.json\n"); return 1; }
if(!prompt){ fprintf(stderr,"--chat needs a user message\n"); return 1; }
if(m.c.bos>=0) ids[np++]=m.c.bos;
np+=chat_build(&T,sysmsg,prompt,think,ids+np,65536-np,sp);
if(getenv("K3_CHAT_IDS")){
fprintf(stderr,"[K3] chat ids:");
for(int i=0;i<np;i++) fprintf(stderr," %d",ids[i]);
fprintf(stderr,"\n");
}
} else if(prompt){
if(!has_tok){ fprintf(stderr,"no tokenizer.json in the snapshot — pass --ids instead\n\n");
k3_usage(argv[0]); return 1; }
if(m.c.bos>=0) ids[np++]=m.c.bos;
np+=tok_encode(&T,prompt,(int)strlen(prompt),ids+np,65536-np);
} else { fprintf(stderr,"no prompt and no --ids: nothing to generate from\n\n");
k3_usage(argv[0]); return 1; }
fprintf(stderr,"[K3] prompt: %d tokens | ngen %d | temp %.2f\n",np,ngen,temp);
int max_t=getenv("K3_MAXT")?atoi(getenv("K3_MAXT")):np+ngen;
kv_alloc(&m,max_t);
if(getenv("K3_LOGITS")){
g_lfp=fopen(getenv("K3_LOGITS"),"wb");
if(!g_lfp){ perror(getenv("K3_LOGITS")); return 1; }
}
int chunk=getenv("K3_CHUNK")?atoi(getenv("K3_CHUNK")):32;
if(chunk<1) chunk=1;
if(chunk>512) chunk=512;
#ifdef COLI_VULKAN
{ int r=k3c_prefill_rows(&m); if(r>0 && !g_lfp) chunk=r; } /* the chain's chunk from the budget: big MoE steps */
#endif
if(m.trace && chunk>1){
chunk=1; /* trace rows are token-major by contract */
fprintf(stderr,"[K3] K3_TRACE set: prefill chunk forced to 1\n");
}
double t0=now_s(); float *lo=NULL;
for(int i=0;i<np;i+=chunk){
int Cc=np-i<chunk?np-i:chunk;
if(lo) free(lo);
lo=step_chunk(&m,ids+i,i,Cc);
fprintf(stderr,"\r[K3] prefill %d/%d (%.1fs)",i+Cc,np,now_s()-t0);
}
if(g_lfp){ fclose(g_lfp); g_lfp=NULL; }
fprintf(stderr,"\n[K3] prefill done in %.1fs (%.2f tok/s)\n",now_s()-t0,np/(now_s()-t0));
if(!m.has_head||!lo){
fprintf(stderr,"[K3] no head — trace written, stopping after prefill\n");
if(m.trace) fclose(m.trace);
if(g_k3_val_fp){ fflush(g_k3_val_fp); fclose(g_k3_val_fp); g_k3_val_fp=NULL; }
if(g_k3_val_lfp){ fflush(g_k3_val_lfp); fclose(g_k3_val_lfp); g_k3_val_lfp=NULL; }
#ifdef COLI_METAL
if(g_k3_metal) coli_metal_shutdown();
#endif
return 0;
}
double tg=now_s(); int ntok=0;
char buf[512];
/* chat print filter: hide the XTML structure, label the channels.
* Structural runs are <|open|>/<|close|> TAGTEXT <|sep|> — suppress them
* and print a channel banner when the response channel opens. */
int xsup=0, xopen=0; char xtag[64]; int xtl=0;
if(chat&&think){ printf("[think] "); fflush(stdout); }
for(int s=0;s<ngen;s++){
int t=sample_tok(lo,m.c.vocab,temp,1.f);
free(lo); lo=NULL;
int is_eos=0; for(int e=0;e<m.c.n_eos;e++) if(t==m.c.eos[e]) is_eos=1;
int show=1;
if(chat&&sp[0]>=0){
if(t==sp[0]||t==sp[1]){ xsup=1; xopen=(t==sp[0]); xtl=0; show=0; }
else if(t==sp[2]){
if(xsup){ xsup=0; xtag[xtl]=0;
if(xopen&&!strcmp(xtag,"response")){ printf("\n\n[response] "); fflush(stdout); }
}
show=0;
} else if(xsup){
if(has_tok){ int n2=tok_decode(&T,&t,1,buf,sizeof(buf)-1);
if(xtl+n2<(int)sizeof(xtag)){ memcpy(xtag+xtl,buf,n2); xtl+=n2; } }
show=0;
} else if(t==sp[3]) show=0;
}
if(show){
if(has_tok){ int n2=tok_decode(&T,&t,1,buf,sizeof(buf)-1); fwrite(buf,1,n2,stdout); fflush(stdout); }
else { printf("%d ",t); fflush(stdout); }
}
ntok++;
if(is_eos){ fprintf(stderr,"\n[K3] eos\n"); break; }
if(np+ntok>=max_t){ fprintf(stderr,"\n[K3] context full\n"); break; }
lo=step_chunk(&m,&t,np+ntok-1,1);
double el=now_s()-tg;
/* Per-token stats every K3_STAT_EVERY tokens (default 32) to keep the token stream
* readable. The final summary below always prints. K3_STAT_EVERY=1 restores per-token. */
static int stat_every=-1;
if(stat_every<0){ const char *e=getenv("K3_STAT_EVERY"); stat_every=e?atoi(e):32; if(stat_every<1) stat_every=1; }
if(ntok % stat_every == 0)
fprintf(stderr," [tok %d: %.1fs/tok, hit %.0f%%, %.1f GB read]\n",
ntok,el/ntok,100.0*m.hits/(m.hits+m.miss+1e-9),m.ebytes/1e9);
}
if(lo) free(lo);
double dt=now_s()-tg;
fprintf(stderr,"\n[K3] decode %d tokens in %.1fs (%.2f tok/s) | expert hit %.1f%% (%llu/%llu) | %.1f GB streamed\n",
ntok,dt,ntok/dt,100.0*m.hits/(m.hits+m.miss+1e-9),
(unsigned long long)m.hits,(unsigned long long)(m.hits+m.miss),m.ebytes/1e9);
fprintf(stderr,"[K3] time: attn %.1fs moe %.1fs (eload %.1fs) head %.1fs | RSS %.1f GB\n",
m.t_attn,m.t_moe,m.t_eload,m.t_head,rss_gb());
#ifdef COLI_VULKAN
k3_vk_report(&m,"run");
#endif
/* One line, every engine, one format: `coli tune` sweeps scheduling knobs and
* needs tokens-and-elapsed to compare candidates. Before this only colibri
* emitted a parseable throughput line (REPLAY decode), so the tuner was
* GLM-only and bannered the right model while launching the wrong engine
* (#898). Printed to stdout, which is what autotune captures.
* Tokens and seconds, not tok/s: the ratio is derived by the caller at full
* precision (#852 -- two decimals of tok/s is one significant digit at the
* rates this engine runs at). */
printf("TUNE decode: %d tokens in %.3fs\n", ntok, dt);
if(m.trace) fclose(m.trace);
if(g_k3_usage[0]) /* USAGE_SAVE=0 honoured inside rt_save (#1039) */
rt_save(g_k3_usage,0); /* same bytes as every other engine */
if(g_k3_val_fp){ fflush(g_k3_val_fp); fclose(g_k3_val_fp); g_k3_val_fp=NULL; }
if(g_k3_val_lfp){ fflush(g_k3_val_lfp); fclose(g_k3_val_lfp); g_k3_val_lfp=NULL; }
#ifdef COLI_METAL
if(g_k3_metal) coli_metal_shutdown();
#endif
return 0;
}
#endif /* KIMI_K3_NO_MAIN */
#ifdef COLI_SEGMENT_ADAPTER
/* ---------- engine-owned Segment adapter ------------------------------ */
typedef struct {
Model model;
uint32_t layer_begin, layer_end, context_tokens, state_width, nbmax;
pthread_mutex_t run_lock;
} KimiSegmentEngine;
typedef struct {
KimiSegmentEngine *engine;
float **kstate, **cwq, **cwk, **cwv;
float **Lc, **Rc, **Ic;
uint32_t context_tokens, position;
} KimiSegmentSession;
static void kimi_segment_layer_destroy(Layer *layer) {
if (!layer) return;
free(layer->in_ln); free(layer->post_ln);
free(layer->attn_sw); free(layer->mlp_sw);
if (layer->kda) {
Kda *a = &layer->a;
w_release_host(&a->q); w_release_host(&a->k);
w_release_host(&a->v); w_release_host(&a->o);
w_release_host(&a->g);
free(a->conv_q); free(a->conv_k); free(a->conv_v);
free(a->fa); free(a->fb); free(a->bp);
free(a->dt); free(a->A); free(a->onw);
} else {
Mla *a = &layer->m;
w_release_host(&a->qa); w_release_host(&a->qb);
w_release_host(&a->kva); w_release_host(&a->kvb);
w_release_host(&a->o); w_release_host(&a->g);
w_release_host(&a->wk); w_release_host(&a->wq);
w_release_host(&a->wp);
free(a->qa_ln); free(a->kva_ln);
free(a->knw); free(a->knb);
}
if (layer->sparse) {
Moe *moe = &layer->moe;
free(moe->router); free(moe->rbias); free(moe->lat_norm);
w_release_host(&moe->lat_down); w_release_host(&moe->lat_up);
w_release_host(&moe->sh_gate); w_release_host(&moe->sh_up);
w_release_host(&moe->sh_down);
} else {
w_release_host(&layer->d_gate); w_release_host(&layer->d_up);
w_release_host(&layer->d_down);
}
}
static void kimi_segment_model_destroy(KimiSegmentEngine *engine) {
if (!engine) return;
Model *model = &engine->model;
for (uint32_t layer = engine->layer_begin; layer < engine->layer_end;
layer++) {
kimi_segment_layer_destroy(&model->L[layer]);
if (model->ecache) {
LCache *cache = &model->ecache[layer];
for (int slot = 0; slot < cache->cap; slot++)
free(cache->s[slot].base);
free(cache->slot_by_expert); free(cache->s);
}
}
for (size_t slot = 0; slot < sizeof(model->ws) / sizeof(model->ws[0]);
slot++)
free(model->ws[slot].base);
free(model->final_norm); free(model->out_sw);
w_release_host(&model->lm_head);
free(model->kstate); free(model->cwq); free(model->cwk); free(model->cwv);
if (model->Lc || model->Rc) {
for (int layer = 0; layer < model->c.n_layers; layer++) {
free(model->Lc ? model->Lc[layer] : NULL);
free(model->Rc ? model->Rc[layer] : NULL);
}
}
free(model->Lc); free(model->Rc);
free(model->dsa_nsel); free(model->dsa_sel);
free(model->eref); free(model->ecache); free(model->L);
kv_prefix_free(&model->kvp);
st_destroy(&model->S);
}
static int kimi_segment_engine_open(
void **engine_impl, ColiSegmentCapabilities *capabilities,
const ColiSegmentEngineOptions *options, char *error, size_t error_size) {
if (!engine_impl || !capabilities || !options)
return coli_segment_adapter_error(error, error_size,
"invalid Kimi K3 Segment open");
*engine_impl = NULL;
if (options->backend_mask &&
(options->backend_mask & ~COLI_SEGMENT_CAP_CPU))
return coli_segment_adapter_error(error, error_size,
"Kimi K3 Segment currently supports CPU");
Cfg config;
memset(&config, 0, sizeof(config));
load_cfg(&config, options->model_dir);
if (options->layer_end > (uint32_t)config.n_layers)
return coli_segment_adapter_error(error, error_size,
"Kimi K3 Segment range exceeds model");
uint64_t nbmax = ((uint64_t)config.n_layers + config.res_bs - 1u) /
(uint64_t)config.res_bs;
uint64_t state_width = (uint64_t)config.hidden * (1u + nbmax);
if (state_width > UINT32_MAX)
return coli_segment_adapter_error(error, error_size,
"Kimi K3 boundary state is too wide");
int sparse_layers = 0;
for (uint32_t layer = options->layer_begin; layer < options->layer_end;
layer++)
if (layer >= (uint32_t)config.first_dense) sparse_layers++;
int expert_cap = 0;
if (options->memory_limit_bytes && sparse_layers) {
uint64_t w1p = (uint64_t)config.moe_inter *
((uint64_t)config.latent / 2u);
uint64_t w1s = (uint64_t)config.moe_inter *
((uint64_t)config.latent / 32u);
uint64_t w2p = (uint64_t)config.latent *
((uint64_t)config.moe_inter / 2u);
uint64_t w2s = (uint64_t)config.latent *
((uint64_t)config.moe_inter / 32u);
uint64_t slot_bytes = 2u * (w1p + w1s) + w2p + w2s;
uint64_t slots = slot_bytes
? options->memory_limit_bytes / slot_bytes /
(uint64_t)sparse_layers : 0;
expert_cap = slots > (uint64_t)config.n_experts
? config.n_experts : (int)slots;
if (expert_cap < 1) expert_cap = 1;
}
KimiSegmentEngine *engine = calloc(1, sizeof(*engine));
if (!engine)
return coli_segment_adapter_error(error, error_size,
"out of memory opening Kimi Segment");
engine->layer_begin = options->layer_begin;
engine->layer_end = options->layer_end;
engine->context_tokens = options->context_tokens;
engine->state_width = (uint32_t)state_width;
engine->nbmax = (uint32_t)nbmax;
if (pthread_mutex_init(&engine->run_lock, NULL)) {
free(engine);
return coli_segment_adapter_error(error, error_size,
"cannot initialize Kimi Segment lock");
}
model_init_range(&engine->model, options->model_dir,
(int)options->layer_begin, (int)options->layer_end,
0, 0, 0, expert_cap, (int)options->context_tokens, 1);
free(engine->model.kstate); free(engine->model.cwq);
free(engine->model.cwk); free(engine->model.cwv);
engine->model.kstate = NULL; engine->model.cwq = NULL;
engine->model.cwk = NULL; engine->model.cwv = NULL;
memset(capabilities, 0, sizeof(*capabilities));
capabilities->struct_size = sizeof(*capabilities);
capabilities->abi_version = COLI_SEGMENT_ABI_VERSION;
capabilities->flags = COLI_SEGMENT_CAP_SNAPSHOT |
COLI_SEGMENT_CAP_RANGE_NATIVE |
COLI_SEGMENT_CAP_MULTI_SESSION |
COLI_SEGMENT_CAP_CPU;
coli_segment_capability_string(capabilities->engine_id,
sizeof(capabilities->engine_id), "kimi");
coli_segment_capability_string(capabilities->state_schema,
sizeof(capabilities->state_schema),
"kimi-k3/attnres-kda-mla-dsa-f32-v1");
int bits = getenv("K3_BITS") ? atoi(getenv("K3_BITS")) : 4;
int mla_bits = getenv("K3_MLA_BITS") ? atoi(getenv("K3_MLA_BITS")) : 8;
snprintf(capabilities->numeric_class,
sizeof(capabilities->numeric_class),
"kimi-k3/q%d-mla%d-mxfp4-idot%d/f32/cpu-v1",
bits, mla_bits, g_k3_idot != 0);
capabilities->state_dtype = COLI_SEGMENT_DTYPE_F32;
capabilities->state_width = engine->state_width;
capabilities->max_batch_rows = 128;
capabilities->max_context_tokens = options->context_tokens;
capabilities->num_layers = (uint32_t)config.n_layers;
*engine_impl = engine;
return 0;
}
static void kimi_segment_engine_destroy(void *engine_impl) {
KimiSegmentEngine *engine = (KimiSegmentEngine *)engine_impl;
if (!engine) return;
kimi_segment_model_destroy(engine);
pthread_mutex_destroy(&engine->run_lock);
free(engine);
}
static void kimi_segment_session_free(KimiSegmentSession *session) {
if (!session) return;
if (session->engine) {
for (uint32_t layer = session->engine->layer_begin;
layer < session->engine->layer_end; layer++) {
free(session->kstate ? session->kstate[layer] : NULL);
free(session->cwq ? session->cwq[layer] : NULL);
free(session->cwk ? session->cwk[layer] : NULL);
free(session->cwv ? session->cwv[layer] : NULL);
free(session->Lc ? session->Lc[layer] : NULL);
free(session->Rc ? session->Rc[layer] : NULL);
free(session->Ic ? session->Ic[layer] : NULL);
}
}
free(session->kstate); free(session->cwq); free(session->cwk);
free(session->cwv); free(session->Lc); free(session->Rc);
free(session->Ic); free(session);
}
static int kimi_segment_session_create(
void *engine_impl, void **session_impl,
const ColiSegmentSessionOptions *options, char *error, size_t error_size) {
KimiSegmentEngine *engine = (KimiSegmentEngine *)engine_impl;
if (!engine || !session_impl || !options)
return coli_segment_adapter_error(error, error_size,
"invalid Kimi Segment session");
*session_impl = NULL;
KimiSegmentSession *session = calloc(1, sizeof(*session));
if (!session)
return coli_segment_adapter_error(error, error_size,
"out of memory creating Kimi session");
session->engine = engine;
session->context_tokens = options->context_tokens;
size_t layers = (size_t)engine->model.c.n_layers;
session->kstate = calloc(layers, sizeof(*session->kstate));
session->cwq = calloc(layers, sizeof(*session->cwq));
session->cwk = calloc(layers, sizeof(*session->cwk));
session->cwv = calloc(layers, sizeof(*session->cwv));
session->Lc = calloc(layers, sizeof(*session->Lc));
session->Rc = calloc(layers, sizeof(*session->Rc));
session->Ic = calloc(layers, sizeof(*session->Ic));
if (!session->kstate || !session->cwq || !session->cwk ||
!session->cwv || !session->Lc || !session->Rc || !session->Ic)
goto oom;
Cfg *config = &engine->model.c;
uint64_t state_bytes = 0;
for (uint32_t layer = engine->layer_begin; layer < engine->layer_end;
layer++) {
size_t cells;
if (config->is_kda[layer]) {
if (coli_segment_size_mul((size_t)config->kda_heads,
(size_t)config->kda_hd, &cells) ||
coli_segment_size_mul(cells, (size_t)config->kda_hd,
&cells)) goto oom;
session->kstate[layer] = calloc(cells, sizeof(float));
if (coli_segment_size_mul((size_t)config->kda_proj,
(size_t)config->conv_k, &cells)) goto oom;
session->cwq[layer] = calloc(cells, sizeof(float));
session->cwk[layer] = calloc(cells, sizeof(float));
session->cwv[layer] = calloc(cells, sizeof(float));
state_bytes += ((uint64_t)config->kda_heads * config->kda_hd *
config->kda_hd + 3u * (uint64_t)cells) *
sizeof(float);
if (!session->kstate[layer] || !session->cwq[layer] ||
!session->cwk[layer] || !session->cwv[layer]) goto oom;
} else {
if (coli_segment_size_mul(options->context_tokens,
(size_t)config->kv_lora, &cells)) goto oom;
session->Lc[layer] = calloc(cells, sizeof(float));
state_bytes += (uint64_t)cells * sizeof(float);
if (coli_segment_size_mul(options->context_tokens,
(size_t)config->qk_rope, &cells)) goto oom;
session->Rc[layer] = calloc(cells, sizeof(float));
state_bytes += (uint64_t)cells * sizeof(float);
if (config->index_hd > 0 && config->idx_type[layer]) {
if (coli_segment_size_mul(options->context_tokens,
(size_t)config->index_hd,
&cells)) goto oom;
session->Ic[layer] = calloc(cells, sizeof(float));
state_bytes += (uint64_t)cells * sizeof(float);
}
if (!session->Lc[layer] || !session->Rc[layer] ||
(config->index_hd > 0 && config->idx_type[layer] &&
!session->Ic[layer])) goto oom;
}
}
if (options->memory_limit_bytes &&
state_bytes > options->memory_limit_bytes) {
kimi_segment_session_free(session);
return coli_segment_adapter_error(error, error_size,
"Kimi session state exceeds memory limit");
}
*session_impl = session;
return 0;
oom:
kimi_segment_session_free(session);
return coli_segment_adapter_error(error, error_size,
"out of memory allocating Kimi state");
}
static void kimi_segment_session_destroy(void *session_impl) {
kimi_segment_session_free((KimiSegmentSession *)session_impl);
}
static int kimi_segment_session_run(void *session_impl,
const ColiSegmentRunRequest *request,
char *error, size_t error_size) {
KimiSegmentSession *session = (KimiSegmentSession *)session_impl;
if (!session || !request || request->position != session->position)
return coli_segment_adapter_error(
error, error_size, "Kimi Segment requires contiguous positions");
if (request->should_cancel &&
request->should_cancel(request->cancel_user_data))
return coli_segment_adapter_error(error, error_size,
"Kimi Segment run cancelled");
KimiSegmentEngine *engine = session->engine;
Cfg *config = &engine->model.c;
size_t hidden_cells, residual_cells;
if (coli_segment_size_mul(request->rows, (size_t)config->hidden,
&hidden_cells) ||
coli_segment_size_mul(hidden_cells, (size_t)engine->nbmax,
&residual_cells))
return coli_segment_adapter_error(error, error_size,
"Kimi boundary state overflows");
float *hidden = calloc(hidden_cells, sizeof(float));
float *residuals = calloc(residual_cells, sizeof(float));
if (!hidden || !residuals) {
free(hidden); free(residuals);
return coli_segment_adapter_error(error, error_size,
"out of memory running Kimi Segment");
}
const float *input = (const float *)request->input;
for (uint32_t row = 0; row < request->rows; row++) {
memcpy(hidden + (size_t)row * config->hidden,
input + (size_t)row * engine->state_width,
(size_t)config->hidden * sizeof(float));
memcpy(residuals + (size_t)row * engine->nbmax * config->hidden,
input + (size_t)row * engine->state_width + config->hidden,
(size_t)engine->nbmax * config->hidden * sizeof(float));
}
pthread_mutex_lock(&engine->run_lock);
Model *model = &engine->model;
model->kstate = session->kstate;
model->cwq = session->cwq; model->cwk = session->cwk;
model->cwv = session->cwv;
model->Lc = session->Lc; model->Rc = session->Rc;
model->max_t = (int)session->context_tokens;
for (uint32_t layer = engine->layer_begin; layer < engine->layer_end;
layer++)
if (!config->is_kda[layer]) model->L[layer].m.Ic = session->Ic[layer];
int nb = ((int)engine->layer_begin + config->res_bs - 1) / config->res_bs;
int cancelled = 0;
int result = k3_layers_forward_range(
model, hidden, residuals, &nb, (int)request->position,
(int)request->rows, (int)engine->layer_begin,
(int)engine->layer_end, NULL, NULL, &cancelled);
for (uint32_t layer = engine->layer_begin; layer < engine->layer_end;
layer++)
if (!config->is_kda[layer]) model->L[layer].m.Ic = NULL;
model->kstate = NULL; model->cwq = NULL; model->cwk = NULL;
model->cwv = NULL; model->Lc = NULL; model->Rc = NULL; model->max_t = 0;
pthread_mutex_unlock(&engine->run_lock);
if (result) {
free(hidden); free(residuals);
return coli_segment_adapter_error(error, error_size,
"Kimi Segment layer run failed");
}
float *output = (float *)request->output;
for (uint32_t row = 0; row < request->rows; row++) {
memcpy(output + (size_t)row * engine->state_width,
hidden + (size_t)row * config->hidden,
(size_t)config->hidden * sizeof(float));
memcpy(output + (size_t)row * engine->state_width + config->hidden,
residuals + (size_t)row * engine->nbmax * config->hidden,
(size_t)engine->nbmax * config->hidden * sizeof(float));
}
free(hidden); free(residuals);
session->position += request->rows;
return 0;
}
static int kimi_segment_spans(
KimiSegmentSession *session, uint32_t position,
ColiSegmentStateSpan **spans_output, size_t *count_output,
char *error, size_t error_size) {
size_t capacity = (size_t)(session->engine->layer_end -
session->engine->layer_begin) * 4u;
ColiSegmentStateSpan *spans = capacity
? calloc(capacity, sizeof(*spans)) : NULL;
if (capacity && !spans)
return coli_segment_adapter_error(error, error_size,
"out of memory describing Kimi state");
Cfg *config = &session->engine->model.c;
size_t count = 0;
for (uint32_t layer = session->engine->layer_begin;
layer < session->engine->layer_end; layer++) {
if (config->is_kda[layer]) {
size_t rec_cells = (size_t)config->kda_heads * config->kda_hd *
config->kda_hd;
size_t conv_cells = (size_t)config->kda_proj * config->conv_k;
spans[count++] = (ColiSegmentStateSpan){
session->kstate[layer], rec_cells * sizeof(float)};
spans[count++] = (ColiSegmentStateSpan){
session->cwq[layer], conv_cells * sizeof(float)};
spans[count++] = (ColiSegmentStateSpan){
session->cwk[layer], conv_cells * sizeof(float)};
spans[count++] = (ColiSegmentStateSpan){
session->cwv[layer], conv_cells * sizeof(float)};
} else {
spans[count++] = (ColiSegmentStateSpan){
session->Lc[layer],
(size_t)position * config->kv_lora * sizeof(float)};
spans[count++] = (ColiSegmentStateSpan){
session->Rc[layer],
(size_t)position * config->qk_rope * sizeof(float)};
if (config->index_hd > 0 && config->idx_type[layer])
spans[count++] = (ColiSegmentStateSpan){
session->Ic[layer],
(size_t)position * config->index_hd * sizeof(float)};
}
}
*spans_output = spans; *count_output = count;
return 0;
}
static int kimi_segment_session_snapshot(
void *session_impl, ColiSegmentWriteFn write_fn, void *write_user_data,
char *error, size_t error_size) {
KimiSegmentSession *session = (KimiSegmentSession *)session_impl;
ColiSegmentStateSpan *spans = NULL;
size_t count = 0, payload_bytes;
if (!session || kimi_segment_spans(session, session->position, &spans,
&count, error, error_size))
return -1;
if (coli_segment_spans_size(spans, count, &payload_bytes)) {
free(spans);
return coli_segment_adapter_error(error, error_size,
"Kimi snapshot size overflow");
}
ColiSegmentSnapshotHeader header;
coli_segment_snapshot_header_init(
&header, "kimi", session->engine->layer_begin,
session->engine->layer_end, session->context_tokens, session->position,
payload_bytes, coli_segment_spans_hash(spans, count));
int result = coli_segment_stream_write(
write_fn, write_user_data, &header, sizeof(header), error, error_size);
if (!result)
result = coli_segment_spans_write(spans, count, write_fn,
write_user_data, error, error_size);
free(spans);
return result;
}
static int kimi_segment_session_restore(
void *session_impl, ColiSegmentReadFn read_fn, void *read_user_data,
char *error, size_t error_size) {
KimiSegmentSession *session = (KimiSegmentSession *)session_impl;
ColiSegmentSnapshotHeader header;
if (!session || coli_segment_stream_read(read_fn, read_user_data, &header,
sizeof(header), error, error_size))
return -1;
ColiSegmentStateSpan *spans = NULL;
size_t count = 0, payload_bytes;
if (kimi_segment_spans(session, header.position, &spans, &count,
error, error_size)) return -1;
if (coli_segment_spans_size(spans, count, &payload_bytes) ||
coli_segment_snapshot_header_valid(
&header, "kimi", session->engine->layer_begin,
session->engine->layer_end, session->context_tokens,
payload_bytes, error, error_size)) {
free(spans); return -1;
}
int result = coli_segment_spans_restore(
spans, count, header.payload_hash, read_fn, read_user_data,
error, error_size);
free(spans);
if (!result) session->position = header.position;
return result;
}
static const ColiSegmentAdapter kimi_segment_adapter = {
sizeof(ColiSegmentAdapter), COLI_SEGMENT_ABI_VERSION, "kimi",
kimi_segment_engine_open, kimi_segment_engine_destroy,
kimi_segment_session_create, kimi_segment_session_destroy,
kimi_segment_session_run, kimi_segment_session_snapshot,
kimi_segment_session_restore, {0}
};
int coli_kimi_segment_adapter_register(void) {
return coli_segment_adapter_register(&kimi_segment_adapter);
}
#endif /* COLI_SEGMENT_ADAPTER */
#ifdef COLI_EDGE_ADAPTER
/* ---------- engine-owned model Edge adapter --------------------------- */
typedef struct {
Model model;
Tok tokenizer;
uint32_t state_width, nbmax;
} KimiEdgeEngine;
static uint64_t kimi_edge_w_bytes(const W *weight) {
uint64_t cells = (uint64_t)weight->O * weight->I;
if (weight->fmt == 0) return cells * sizeof(float);
if (weight->fmt == 1) return cells + (uint64_t)weight->O * sizeof(float);
if (weight->fmt == 2)
return (cells + 1u) / 2u + (uint64_t)weight->O * sizeof(float);
if (weight->fmt == 4) {
uint64_t groups = ((uint64_t)weight->I + weight->gs - 1u) /
(uint64_t)weight->gs;
return (cells + 1u) / 2u +
(uint64_t)weight->O * groups * sizeof(float);
}
return 0;
}
static void kimi_edge_engine_destroy(void *engine_impl) {
KimiEdgeEngine *engine = (KimiEdgeEngine *)engine_impl;
if (!engine) return;
free(engine->model.final_norm); free(engine->model.out_sw);
w_release_host(&engine->model.lm_head);
st_destroy(&engine->model.S);
tok_free(&engine->tokenizer);
free(engine);
}
static int kimi_edge_engine_open(
void **engine_impl, ColiEdgeCapabilities *capabilities,
const ColiEdgeEngineOptions *options, char *error, size_t error_size) {
if (!engine_impl || !capabilities || !options)
return coli_edge_adapter_error(error, error_size,
"invalid Kimi K3 Edge open");
*engine_impl = NULL;
if (options->backend_mask &&
(options->backend_mask & ~COLI_EDGE_CAP_CPU))
return coli_edge_adapter_error(error, error_size,
"Kimi K3 Edge supports CPU only");
KimiEdgeEngine *engine = calloc(1, sizeof(*engine));
if (!engine)
return coli_edge_adapter_error(error, error_size,
"out of memory opening Kimi K3 Edge");
Model *model = &engine->model;
load_cfg(&model->c, options->model_dir);
st_init_multi(&model->S, options->model_dir, getenv("K3_DIRS"));
model->pfx[0] = '\0';
if (!st_has(&model->S, "model.layers.0.input_layernorm.weight") &&
st_has(&model->S,
"language_model.model.layers.0.input_layernorm.weight"))
snprintf(model->pfx, sizeof(model->pfx), "language_model.");
Cfg *config = &model->c;
uint64_t nbmax = ((uint64_t)config->n_layers + config->res_bs - 1u) /
(uint64_t)config->res_bs;
uint64_t state_width = (1u + nbmax) * (uint64_t)config->hidden;
if (!nbmax || state_width > UINT32_MAX) {
kimi_edge_engine_destroy(engine);
return coli_edge_adapter_error(error, error_size,
"Kimi K3 boundary state is too wide");
}
engine->nbmax = (uint32_t)nbmax;
engine->state_width = (uint32_t)state_width;
g_bits_env = getenv("K3_BITS") != NULL;
g_k3_mmap = getenv("K3_MMAP") ? atoi(getenv("K3_MMAP")) : 0;
g_k3_idot = getenv("K3_IDOT") ? atoi(getenv("K3_IDOT")) : 1;
int head_bits = getenv("K3_HEAD_BITS")
? atoi(getenv("K3_HEAD_BITS")) : 8;
char name[512];
snprintf(name, sizeof(name), "%smodel.norm.weight", model->pfx);
if (!st_has(&model->S, name)) {
kimi_edge_engine_destroy(engine);
return coli_edge_adapter_error(error, error_size,
"Kimi K3 final head is not present");
}
model->has_head = 1;
model->final_norm = f32_load(model, "model.norm.weight", config->hidden);
float *residual_norm = f32_load(
model, "model.output_attn_res_norm.weight", config->hidden);
float *residual_projection = f32_load(
model, "model.output_attn_res_proj.weight", config->hidden);
model->out_sw = falloc(config->hidden);
for (int item = 0; item < config->hidden; item++)
model->out_sw[item] = residual_norm[item] * residual_projection[item];
free(residual_norm); free(residual_projection);
w_load(model, &model->lm_head, "lm_head.weight",
config->vocab, config->hidden, head_bits);
char tokenizer_path[4096];
snprintf(tokenizer_path, sizeof(tokenizer_path), "%s/tokenizer.json",
options->model_dir);
tok_load(&engine->tokenizer, tokenizer_path);
uint64_t resident = kimi_edge_w_bytes(&model->lm_head) +
2u * (uint64_t)config->hidden * sizeof(float);
if (options->memory_limit_bytes && resident > options->memory_limit_bytes) {
kimi_edge_engine_destroy(engine);
return coli_edge_adapter_error(error, error_size,
"Kimi K3 Edge exceeds memory limit");
}
int bits = getenv("K3_BITS") ? atoi(getenv("K3_BITS")) : 4;
int mla_bits = getenv("K3_MLA_BITS") ? atoi(getenv("K3_MLA_BITS")) : 8;
memset(capabilities, 0, sizeof(*capabilities));
capabilities->struct_size = sizeof(*capabilities);
capabilities->abi_version = COLI_EDGE_ABI_VERSION;
capabilities->flags = COLI_EDGE_CAP_TOKENIZE |
COLI_EDGE_CAP_DETOKENIZE |
COLI_EDGE_CAP_GREEDY | COLI_EDGE_CAP_LOGITS |
COLI_EDGE_CAP_CPU;
coli_edge_capability_string(capabilities->engine_id,
sizeof(capabilities->engine_id), "kimi");
coli_edge_capability_string(capabilities->state_schema,
sizeof(capabilities->state_schema),
"kimi-k3/attnres-kda-mla-dsa-f32-v1");
snprintf(capabilities->numeric_class,
sizeof(capabilities->numeric_class),
"kimi-k3/q%d-mla%d-mxfp4-idot%d/f32/cpu-v1",
bits, mla_bits, g_k3_idot != 0);
coli_edge_capability_string(capabilities->tokenizer_class,
sizeof(capabilities->tokenizer_class),
"kimi-k3/rank-byte-bpe-v1");
capabilities->state_dtype = COLI_EDGE_DTYPE_F32;
capabilities->state_width = engine->state_width;
capabilities->vocab_size = (uint32_t)config->vocab;
capabilities->max_batch_rows = 128;
capabilities->max_context_tokens = UINT32_MAX;
capabilities->num_layers = (uint32_t)config->n_layers;
capabilities->bos_token_id = config->bos;
capabilities->eos_token_id = config->n_eos ? config->eos[0] : -1;
capabilities->resident_bytes = resident;
*engine_impl = engine;
return 0;
}
static int kimi_edge_tokenize(
void *engine_impl, const char *text, size_t text_bytes,
int32_t *token_ids, size_t token_capacity, size_t *token_count,
char *error, size_t error_size) {
KimiEdgeEngine *engine = (KimiEdgeEngine *)engine_impl;
return coli_edge_tok_tokenize(&engine->tokenizer, text, text_bytes,
token_ids, token_capacity, token_count,
error, error_size);
}
static int kimi_edge_detokenize(
void *engine_impl, const int32_t *token_ids, size_t token_count,
char *text, size_t text_capacity, size_t *text_bytes,
char *error, size_t error_size) {
KimiEdgeEngine *engine = (KimiEdgeEngine *)engine_impl;
return coli_edge_tok_detokenize(&engine->tokenizer, token_ids, token_count,
text, text_capacity, text_bytes,
error, error_size);
}
static int kimi_edge_embed(void *engine_impl,
const ColiEdgeEmbedRequest *request,
char *error, size_t error_size) {
KimiEdgeEngine *engine = (KimiEdgeEngine *)engine_impl;
Cfg *config = &engine->model.c;
float *output = (float *)request->output;
char name[512];
snprintf(name, sizeof(name), "%smodel.embed_tokens.weight",
engine->model.pfx);
for (uint32_t row = 0; row < request->rows; row++) {
int token = request->token_ids[row];
if (token < 0 || token >= config->vocab)
return coli_edge_adapter_error(error, error_size,
"Kimi K3 token ID is out of range");
float *state = output + (size_t)row * engine->state_width;
memset(state, 0, (size_t)engine->state_width * sizeof(float));
st_read_slice_f32(&engine->model.S, name,
(int64_t)token * config->hidden,
config->hidden, state, 0);
}
return 0;
}
static int kimi_edge_select(void *engine_impl,
const ColiEdgeSelectRequest *request,
char *error, size_t error_size) {
KimiEdgeEngine *engine = (KimiEdgeEngine *)engine_impl;
Cfg *config = &engine->model.c;
float *mixed = falloc(config->hidden);
float *logits = falloc(config->vocab);
const float *input = (const float *)request->input;
for (uint32_t row = 0; row < request->rows; row++) {
if (request->should_cancel &&
request->should_cancel(request->cancel_user_data)) {
free(logits); free(mixed);
return coli_edge_adapter_error(error, error_size,
"Kimi K3 Edge selection cancelled");
}
const float *state = input + (size_t)row * engine->state_width;
res_mix(mixed, state, state + config->hidden,
(int)engine->nbmax, config->hidden,
engine->model.out_sw, config->eps);
rmsnorm_(mixed, mixed, engine->model.final_norm,
config->hidden, config->eps);
w_matmul(logits, mixed, &engine->model.lm_head, 1);
if (coli_edge_argmax(logits, (uint32_t)config->vocab,
&request->token_ids[row],
request->scores ? &request->scores[row] : NULL)) {
free(logits); free(mixed);
return coli_edge_adapter_error(error, error_size,
"Kimi K3 Edge head failed");
}
}
free(logits); free(mixed);
return 0;
}
static int kimi_edge_logits(void *engine_impl,
const ColiEdgeLogitsRequest *request,
char *error, size_t error_size) {
KimiEdgeEngine *engine = (KimiEdgeEngine *)engine_impl;
Cfg *config = &engine->model.c;
float *mixed = falloc(config->hidden);
const float *input = (const float *)request->input;
for (uint32_t row = 0; row < request->rows; row++) {
if (request->should_cancel &&
request->should_cancel(request->cancel_user_data)) {
free(mixed);
return coli_edge_adapter_error(error, error_size,
"Kimi K3 Edge logits cancelled");
}
const float *state = input + (size_t)row * engine->state_width;
res_mix(mixed, state, state + config->hidden,
(int)engine->nbmax, config->hidden,
engine->model.out_sw, config->eps);
rmsnorm_(mixed, mixed, engine->model.final_norm,
config->hidden, config->eps);
w_matmul(request->logits + (size_t)row * config->vocab,
mixed, &engine->model.lm_head, 1);
}
free(mixed);
return 0;
}
static const ColiEdgeAdapter kimi_edge_adapter = {
sizeof(ColiEdgeAdapter), COLI_EDGE_ABI_VERSION, "kimi",
kimi_edge_engine_open, kimi_edge_engine_destroy,
kimi_edge_tokenize, kimi_edge_detokenize,
kimi_edge_embed, kimi_edge_select, kimi_edge_logits, {0}
};
int coli_kimi_edge_adapter_register(void) {
return coli_edge_adapter_register(&kimi_edge_adapter);
}
#endif /* COLI_EDGE_ADAPTER */