Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
kimi: keep CUDA SiTU expert intermediates on device
ZacharyZcRcc27d9873d30
diff --git a/c/.gitignore b/c/.gitignoreindex 36d9678c..ef2a31d6 100644--- a/c/.gitignore+++ b/c/.gitignore@@ -21,6 +21,7 @@ tests/fuzz_ranstests/bench_omp_graintests/mxfp4_ref.omxfp4_cuda_test+mxfp4_expert_cuda_testabsorb_determinism_testcuda_fmt_trap_testtests/*.dSYM/diff --git a/c/Makefile b/c/Makefileindex 4b7bf644..2ceddc64 100644--- a/c/Makefile+++ b/c/Makefile@@ -956,7 +956,7 @@ rans: $(RANSLIB)$(RANSLIB): tools/rans_ctypes.c rans.h$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)-cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu tests/test_cuda_fmt_trap_cuda.cu tests/test_alloc_footprint_cuda.cu+cuda-test: tests/test_mxfp4_expert_cuda.cu backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu tests/test_cuda_fmt_trap_cuda.cu tests/test_alloc_footprint_cuda.cu@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)./backend_cuda_test$(EXE)@@ -1023,6 +1023,8 @@ cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backen# CUDA box to run the suite."$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_mxfp4_cuda.cu $(if $(filter 1,$(HIP)),-x none) tests/mxfp4_ref.o -o mxfp4_cuda_test$(EXE) $(GPU_TEST_LIBS)./mxfp4_cuda_test$(EXE)+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_mxfp4_expert_cuda.cu $(if $(filter 1,$(HIP)),-x none) tests/mxfp4_ref.o -o mxfp4_expert_cuda_test$(EXE) $(GPU_TEST_LIBS)+ ./mxfp4_expert_cuda_test$(EXE)# Allocator footprint (#687): what a cudaMalloc really takes off the card,# which is what the expert tier has to be charged rather than the logical# byte count. Runs LAST on purpose - it is the only test here that@@ -2172,3 +2174,7 @@ tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.htests/test_kimi_request_state$(EXE): tests/test_kimi_request_state.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h idot.h omp_tune.h route_trace.h$(CC) $(NOCUDA_CFLAGS) tests/test_kimi_request_state.c $(VK_OBJ) -o tests/test_kimi_request_state$(EXE) $(NOCUDA_LDFLAGS)++# Kimi CUDA dispatch/fallback with a fake backend; no CUDA toolkit required.+tests/test_kimi_cuda_expert$(EXE): tests/test_kimi_cuda_expert.c kimi_k3.c backend_cuda.h kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h idot.h omp_tune.h route_trace.h+ $(CC) $(NOCUDA_CFLAGS) -DCOLI_CUDA $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 07d5dfc4..41bf20d9 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -634,6 +634,14 @@ __global__ static void silu_mul(float *gate, const float *up, size_t n) {}}+__global__ static void situ_mul(float *gate, const float *up, size_t n, float b1, float b2) {+ size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;+ if (i < n) {+ float g = gate[i], u = up[i];+ gate[i] = b1 * tanhf(g / b1) * (1.f / (1.f + expf(-g))) * b2 * tanhf(u / b2);+ }+}+/* Four warps share one A tile and compute 16x64 outputs. This matters for* prefill: the first prototype reloaded/converter A once per 16 output cols. */__global__ static void w4a16_matmul(float *y,const float *x,const uint8_t *w,@@ -1748,6 +1756,50 @@ extern "C" int coli_cuda_matmul_mxfp4(float *y, const float *x,return ok;}+/* Reuse one weight/scale staging allocation across the three projections.+ * Default-stream copies are ordered after the previous projection's reads. */+static int mxfp4_project(float *y, const float *x, uint8_t *dw, uint8_t *ds,+ const uint8_t *w, const uint8_t *sc, int S, int I, int O) {+ size_t rb = ((size_t)I + 1) / 2, ng = ((size_t)I + 31) / 32;+ if (!cuda_ok(cudaMemcpy(dw, w, (size_t)O * rb, cudaMemcpyHostToDevice), "expert weight upload") ||+ !cuda_ok(cudaMemcpy(ds, sc, (size_t)O * ng, cudaMemcpyHostToDevice), "expert scale upload")) return 0;+ quant_matmul<<<dim3(O, S), 256>>>(y, x, dw, reinterpret_cast<const float *>(ds),+ 7, S, I, O, rb, 32, (int)ng);+ return cuda_ok(cudaGetLastError(), "MXFP4 expert projection");+}++extern "C" int coli_cuda_expert_mxfp4(float *y, const float *x,+ const unsigned char *gate_w, const unsigned char *gate_s,+ const unsigned char *up_w, const unsigned char *up_s,+ const unsigned char *down_w, const unsigned char *down_s,+ int S, int D, int I, float b1, float b2) {+ if (fault_injected() || !x || !y || !gate_w || !gate_s || !up_w || !up_s ||+ !down_w || !down_s || S < 1 || S > 65535 || D < 1 || I < 1 ||+ !(b1 > 0.f) || !(b2 > 0.f) || !std::isfinite(b1) || !std::isfinite(b2)) return 0;+ DeviceContext *ctx = find_ctx(0);+ if (!select_ctx(ctx)) return 0;+ size_t xb = (size_t)S * D * sizeof(float), ib = (size_t)S * I * sizeof(float);+ if (!reserve(&ctx->x, &ctx->x_cap, xb) || !reserve(&ctx->y, &ctx->y_cap, xb) ||+ !reserve(&ctx->gate, &ctx->gate_cap, ib) || !reserve(&ctx->up, &ctx->up_cap, ib)) return 0;+ size_t gw = (size_t)I * (((size_t)D + 1) / 2), dwb = (size_t)D * (((size_t)I + 1) / 2);+ size_t gs = (size_t)I * (((size_t)D + 31) / 32), dsb = (size_t)D * (((size_t)I + 31) / 32);+ uint8_t *dw = nullptr, *ds = nullptr;+ if (!cuda_ok(cudaMalloc(&dw, gw > dwb ? gw : dwb), "expert weight scratch")) return 0;+ if (!cuda_ok(cudaMalloc(&ds, gs > dsb ? gs : dsb), "expert scale scratch")) { cudaFree(dw); return 0; }+ int ok = cuda_ok(cudaMemcpy(ctx->x, x, xb, cudaMemcpyHostToDevice), "expert input upload") &&+ mxfp4_project(ctx->gate, ctx->x, dw, ds, gate_w, gate_s, S, D, I) &&+ mxfp4_project(ctx->up, ctx->x, dw, ds, up_w, up_s, S, D, I);+ if (ok) {+ size_t n = (size_t)S * I;+ situ_mul<<<(unsigned)((n + 255) / 256), 256>>>(ctx->gate, ctx->up, n, b1, b2);+ ok = cuda_ok(cudaGetLastError(), "SiTU-GLU launch") &&+ mxfp4_project(ctx->y, ctx->gate, dw, ds, down_w, down_s, S, I, D) &&+ cuda_ok(cudaMemcpy(y, ctx->y, xb, cudaMemcpyDeviceToHost), "expert output download");+ }+ cudaFree(dw); cudaFree(ds);+ return ok;+}+extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,ColiCudaTensor *down, float *y,const float *x, int S) {diff --git a/c/backend_cuda.h b/c/backend_cuda.hindex 953e8b39..d2c8b92d 100644--- a/c/backend_cuda.h+++ b/c/backend_cuda.h@@ -115,6 +115,15 @@ COLI_CUDA_DLLEXPORT int coli_cuda_matmul_mxfp4(float *y, const float *x,const unsigned char *e8s,int S, int I, int O);+/* Streaming Kimi expert: down(SiTU(gate(x), up(x))). Weights are MXFP4+ * host buffers; intermediate activations remain on device. No weight cache.+ * Returns 0 on failure; callers must accumulate y only after success. */+COLI_CUDA_DLLEXPORT int coli_cuda_expert_mxfp4(float *y, const float *x,+ const unsigned char *gate_w, const unsigned char *gate_s,+ const unsigned char *up_w, const unsigned char *up_s,+ const unsigned char *down_w, const unsigned char *down_s,+ int S, int D, int I, float b1, float b2);+COLI_CUDA_DLLEXPORT int coli_cuda_matmul(ColiCudaTensor **tensor,float *y, const float *x,const void *weights, const float *scales,diff --git a/c/backend_loader.c b/c/backend_loader.cindex d877553a..a3d4db20 100644--- a/c/backend_loader.c+++ b/c/backend_loader.c@@ -102,6 +102,11 @@ typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const floint fmt, int S, int I, int O, int device, int gs);typedef int (*fn_matmul_mxfp4)(float *y, const float *x, const unsigned char *q4,const unsigned char *e8s, int S, int I, int O);+typedef int (*fn_expert_mxfp4)(float *y, const float *x,+ const unsigned char *gate_w, const unsigned char *gate_s,+ const unsigned char *up_w, const unsigned char *up_s,+ const unsigned char *down_w, const unsigned char *down_s,+ int S, int D, int I, float b1, float b2);typedef void (*fn_tensor_free)(ColiCudaTensor *tensor);typedef size_t (*fn_tensor_bytes)(const ColiCudaTensor *tensor);typedef size_t (*fn_tensor_vram)(const ColiCudaTensor *tensor);@@ -179,6 +184,7 @@ static struct {fn_fp8_set_lut fp8_set_lut;fn_matmul matmul;fn_matmul_mxfp4 matmul_mxfp4;+ fn_expert_mxfp4 expert_mxfp4;fn_tensor_free tensor_free;fn_tensor_bytes tensor_bytes;fn_tensor_vram tensor_vram;@@ -1433,6 +1439,7 @@ static int coli_cuda_load(void){* nothing by this name, and the wrapper's 0 is the engine's own "fall back* to CPU" result, so an older DLL still serves GLM and Qwen3.6 (#1405). */RESOLVE_OPT(matmul_mxfp4, fn_matmul_mxfp4)+ RESOLVE_OPT(expert_mxfp4, fn_expert_mxfp4)RESOLVE_OPT(available_device_count, fn_available_device_count) /* qwen36 tier (#1533); older DLLs fall back to device_count */RESOLVE(tensor_free, fn_tensor_free)RESOLVE(tensor_bytes, fn_tensor_bytes)@@ -1656,6 +1663,15 @@ int coli_cuda_matmul_mxfp4(float *y, const float *x, const unsigned char *q4,return g_cuda.matmul_mxfp4(y, x, q4, e8s, S, I, O);}+int coli_cuda_expert_mxfp4(float *y, const float *x,+ const unsigned char *gate_w, const unsigned char *gate_s,+ const unsigned char *up_w, const unsigned char *up_s,+ const unsigned char *down_w, const unsigned char *down_s,+ int S, int D, int I, float b1, float b2) {+ if (!g_cuda.available || !g_cuda.expert_mxfp4) return 0;+ return g_cuda.expert_mxfp4(y, x, gate_w, gate_s, up_w, up_s, down_w, down_s, S, D, I, b1, b2);+}+void coli_cuda_tensor_free(ColiCudaTensor *tensor){if(g_cuda.available && g_cuda.tensor_free) g_cuda.tensor_free(tensor);}diff --git a/c/kimi_k3.c b/c/kimi_k3.cindex 5543ad61..9dd73357 100644--- a/c/kimi_k3.c+++ b/c/kimi_k3.c@@ -1612,28 +1612,22 @@ static inline float situf_(float g, float u, float b1, float b2){}#ifdef COLI_CUDA-/* CUDA apply for one expert, decode only (S==1).- *- * Same shape as the Vulkan path below and the CPU expert_apply above -- w1/w3,- * SiTU-GLU on the host, then w2 down -- but stateless: the routed tier streams,- * so there is nothing resident to keep a device handle for. Weights ride up- * with the call.- *- * Returns 0 with u untouched on ANY failure, so the caller falls through to the- * disk+CPU path exactly as it does when Vulkan declines. That is the contract- * vLLM's MXFP4 backends use too -- FlashInfer/AITER when they can, an emulation- * path when they cannot -- and it is what makes the fast path safe to attempt- * unconditionally. */+/* Keep SiTU-GLU and intermediate activations on the GPU when supported.+ * Older DLLs lack the optional fused entry point, so retain the three-matmul+ * path as fallback. Accumulate into u only after a complete expert succeeds. */static int cuda_expert_apply(Model *m, const uint8_t *w1p, const uint8_t *w1s,const uint8_t *w2p, const uint8_t *w2s,const uint8_t *w3p, const uint8_t *w3s,const float *z, float wk,float *u, float *gate, float *up, float *hz){Cfg *c=&m->c;- if(!coli_cuda_matmul_mxfp4(gate,z,w1p,w1s,1,c->latent,c->moe_inter)) return 0;- if(!coli_cuda_matmul_mxfp4(up, z,w3p,w3s,1,c->latent,c->moe_inter)) return 0;- for(int i=0;i<c->moe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);- if(!coli_cuda_matmul_mxfp4(hz,gate,w2p,w2s,1,c->moe_inter,c->latent)) return 0;+ if(!coli_cuda_expert_mxfp4(hz,z,w1p,w1s,w3p,w3s,w2p,w2s,+ 1,c->latent,c->moe_inter,c->situ_b1,c->situ_b2)) {+ if(!coli_cuda_matmul_mxfp4(gate,z,w1p,w1s,1,c->latent,c->moe_inter)) return 0;+ if(!coli_cuda_matmul_mxfp4(up, z,w3p,w3s,1,c->latent,c->moe_inter)) return 0;+ for(int i=0;i<c->moe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);+ if(!coli_cuda_matmul_mxfp4(hz,gate,w2p,w2s,1,c->moe_inter,c->latent)) return 0;+ }for(int i=0;i<c->latent;i++) u[i]+=wk*hz[i];return 1;}diff --git a/c/tests/test_kimi_cuda_expert.c b/c/tests/test_kimi_cuda_expert.cnew file mode 100644index 00000000..6815e0f5--- /dev/null+++ b/c/tests/test_kimi_cuda_expert.c@@ -0,0 +1,39 @@+/* Exercise the actual engine dispatch without a GPU, including an older DLL. */+#define main kimi_main_unused+#include "../kimi_k3.c"+#undef main++static int fused_ok, legacy_ok, fused_calls, legacy_calls;+int coli_cuda_init(const int *devices, int n) { return 1; }+int coli_cuda_expert_mxfp4(float *y, const float *x,+ const unsigned char *gw, const unsigned char *gs,+ const unsigned char *uw, const unsigned char *us,+ const unsigned char *dw, const unsigned char *ds,+ int S, int D, int I, float b1, float b2) {+ fused_calls++;+ for (int i = 0; i < D; i++) y[i] = fused_ok ? 2.f : -999.f;+ return fused_ok;+}+int coli_cuda_matmul_mxfp4(float *y, const float *x, const unsigned char *q,+ const unsigned char *sc, int S, int I, int O) {+ legacy_calls++;+ for (int i = 0; i < O; i++) y[i] = 3.f;+ return legacy_ok;+}+int main(void) {+ Model m = {0}; m.c.latent = 3; m.c.moe_inter = 5;+ m.c.situ_b1 = 2.f; m.c.situ_b2 = 3.f;+ uint8_t w[16] = {0}, sc[5] = {0};+ float x[3] = {1,2,3}, out[3] = {4,4,4}, gate[5], up[5], hz[3];+ fused_ok = 1;+ if (!cuda_expert_apply(&m,w,sc,w,sc,w,sc,x,0.5f,out,gate,up,hz) ||+ fused_calls != 1 || legacy_calls || out[0] != 5.f || out[2] != 5.f) return 1;+ fused_ok = 0; legacy_ok = 1;+ if (!cuda_expert_apply(&m,w,sc,w,sc,w,sc,x,0.5f,out,gate,up,hz) ||+ legacy_calls != 3 || out[0] != 6.5f || out[2] != 6.5f) return 1;+ legacy_ok = 0;+ if (cuda_expert_apply(&m,w,sc,w,sc,w,sc,x,0.5f,out,gate,up,hz) ||+ out[0] != 6.5f || out[2] != 6.5f) return 1;+ puts("ok Kimi CUDA fused dispatch, legacy fallback, failed expert does not accumulate");+ return 0;+}diff --git a/c/tests/test_mxfp4_expert_cuda.cu b/c/tests/test_mxfp4_expert_cuda.cunew file mode 100644index 00000000..7d9e67e2--- /dev/null+++ b/c/tests/test_mxfp4_expert_cuda.cu@@ -0,0 +1,55 @@+/* Streaming SiTU-GLU expert vs the CPU MXFP4 decoder, including odd widths. */+#include <cmath>+#include <cstdio>+#include <vector>+#if defined(__HIPCC__)+#include "../backend_gpu_compat.h"+#else+#include <cuda_runtime.h>+#endif+#include "../backend_cuda.h"+extern "C" void mxfp4_ref(float *, const float *, const unsigned char *, const unsigned char *, int, int, int);++static int check_case(int S, int D, int I, float b1, float b2) {+ std::vector<unsigned char> gw((size_t)I * ((D + 1) / 2)), uw(gw.size()), dw((size_t)D * ((I + 1) / 2));+ std::vector<unsigned char> gs((size_t)I * ((D + 31) / 32)), us(gs.size()), ds((size_t)D * ((I + 31) / 32));+ std::vector<float> x((size_t)S * D), g((size_t)S * I), u(g.size()), want(x.size()), got(x.size());+ for (size_t j = 0; j < gw.size(); j++) { gw[j] = (unsigned char)(j * 13 + 2); uw[j] = (unsigned char)(j * 7 + 5); }+ for (size_t j = 0; j < dw.size(); j++) dw[j] = (unsigned char)(j * 11 + 3);+ for (size_t j = 0; j < gs.size(); j++) { gs[j] = 121 + j % 4; us[j] = 122 + j % 3; }+ for (size_t j = 0; j < ds.size(); j++) ds[j] = 123 + j % 4;+ for (size_t j = 0; j < x.size(); j++) x[j] = (int(j % 17) - 8) * 0.125f;+ for (int pass = 0; pass < 2; pass++) {+ mxfp4_ref(g.data(), x.data(), gw.data(), gs.data(), S, D, I);+ mxfp4_ref(u.data(), x.data(), uw.data(), us.data(), S, D, I);+ for (size_t j = 0; j < g.size(); j++)+ g[j] = b1 * tanhf(g[j] / b1) * (1.f / (1.f + expf(-g[j]))) * b2 * tanhf(u[j] / b2);+ mxfp4_ref(want.data(), g.data(), dw.data(), ds.data(), S, I, D);+ if (!coli_cuda_expert_mxfp4(got.data(), x.data(), gw.data(), gs.data(), uw.data(), us.data(),+ dw.data(), ds.data(), S, D, I, b1, b2)) return 1;+ for (size_t j = 0; j < got.size(); j++) {+ if (!std::isfinite(got[j]) || fabsf(got[j] - want[j]) > 2e-5f + 2e-4f * fabsf(want[j])) {+ printf("FAIL S=%d D=%d I=%d row-element=%zu want=%g got=%g\n", S, D, I, j, want[j], got[j]);+ return 1;+ }+ }+ /* Reused host addresses now contain different expert bytes. */+ for (size_t j = 0; j < gw.size(); j++) gw[j] ^= 0x88;+ for (size_t j = 0; j < ds.size(); j++) ds[j]++;+ }+ got[0] = 123.f;+ if (coli_cuda_expert_mxfp4(got.data(), x.data(), gw.data(), gs.data(), uw.data(), us.data(),+ dw.data(), ds.data(), S, D, I, 0.f, b2) || got[0] != 123.f) return 1;+ printf("ok SiTU expert S=%d D=%d I=%d b1=%g b2=%g\n", S, D, I, b1, b2);+ return 0;+}+int main() {+ int n = 0, device = 0;+ if (cudaGetDeviceCount(&n) != cudaSuccess || !n) { puts("SKIP no CUDA device"); return 0; }+ if (!coli_cuda_init(&device, 1)) return 1;+ int failed = check_case(1, 64, 128, 1.5f, 2.5f) |+ check_case(3, 33, 65, 4.f, 3.f) |+ check_case(2, 129, 31, 0.5f, 0.75f);+ coli_cuda_shutdown();+ return failed;+}diff --git a/c/tools/clean.py b/c/tools/clean.pyindex c309212a..ce08c299 100644--- a/c/tools/clean.py+++ b/c/tools/clean.py@@ -27,6 +27,7 @@ FILES = ["iobench", "iobench.exe","backend_cuda.o", "backend_loader.o","backend_cuda_test", "backend_cuda_test.exe",+ "mxfp4_expert_cuda_test", "mxfp4_expert_cuda_test.exe","backend_cuda_bench", "backend_cuda_bench.exe","backend_metal.o", "backend_metal_test","coli_cuda.dll", "coli_cuda.lib", "coli_cuda.exp",