Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

kimi: keep CUDA SiTU expert intermediates on device

ZacharyZcRcc27d9873d30
diff --git a/c/.gitignore b/c/.gitignore
index 36d9678c..ef2a31d6 100644
--- a/c/.gitignore
+++ b/c/.gitignore
@@ -21,6 +21,7 @@ tests/fuzz_rans
tests/bench_omp_grain
tests/mxfp4_ref.o
mxfp4_cuda_test
+mxfp4_expert_cuda_test
absorb_determinism_test
cuda_fmt_trap_test
tests/*.dSYM/
diff --git a/c/Makefile b/c/Makefile
index 4b7bf644..2ceddc64 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -956,7 +956,7 @@ rans: $(RANSLIB)
$(RANSLIB): tools/rans_ctypes.c rans.h
$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)
-cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu tests/test_cuda_fmt_trap_cuda.cu tests/test_alloc_footprint_cuda.cu
+cuda-test: tests/test_mxfp4_expert_cuda.cu backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_absorb_determinism.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c tests/test_fp8_warp_cuda.cu tests/test_fp8_cuda.cu tests/test_weights_owned_cuda.cu tests/test_cuda_fmt_trap_cuda.cu tests/test_alloc_footprint_cuda.cu
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS) $(GPU_TEST_LIBS)
./backend_cuda_test$(EXE)
@@ -1023,6 +1023,8 @@ cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backen
# CUDA box to run the suite.
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_mxfp4_cuda.cu $(if $(filter 1,$(HIP)),-x none) tests/mxfp4_ref.o -o mxfp4_cuda_test$(EXE) $(GPU_TEST_LIBS)
./mxfp4_cuda_test$(EXE)
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_mxfp4_expert_cuda.cu $(if $(filter 1,$(HIP)),-x none) tests/mxfp4_ref.o -o mxfp4_expert_cuda_test$(EXE) $(GPU_TEST_LIBS)
+ ./mxfp4_expert_cuda_test$(EXE)
# Allocator footprint (#687): what a cudaMalloc really takes off the card,
# which is what the expert tier has to be charged rather than the logical
# byte count. Runs LAST on purpose - it is the only test here that
@@ -2172,3 +2174,7 @@ tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.h
tests/test_kimi_request_state$(EXE): tests/test_kimi_request_state.c kimi_k3.c kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h idot.h omp_tune.h route_trace.h
$(CC) $(NOCUDA_CFLAGS) tests/test_kimi_request_state.c $(VK_OBJ) -o tests/test_kimi_request_state$(EXE) $(NOCUDA_LDFLAGS)
+
+# Kimi CUDA dispatch/fallback with a fake backend; no CUDA toolkit required.
+tests/test_kimi_cuda_expert$(EXE): tests/test_kimi_cuda_expert.c kimi_k3.c backend_cuda.h kv_prefix.h serve_codec.h st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h idot.h omp_tune.h route_trace.h
+ $(CC) $(NOCUDA_CFLAGS) -DCOLI_CUDA $< $(VK_OBJ) -o $@ $(NOCUDA_LDFLAGS)
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 07d5dfc4..41bf20d9 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -634,6 +634,14 @@ __global__ static void silu_mul(float *gate, const float *up, size_t n) {
}
}
+__global__ static void situ_mul(float *gate, const float *up, size_t n, float b1, float b2) {
+ size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
+ if (i < n) {
+ float g = gate[i], u = up[i];
+ gate[i] = b1 * tanhf(g / b1) * (1.f / (1.f + expf(-g))) * b2 * tanhf(u / b2);
+ }
+}
+
/* Four warps share one A tile and compute 16x64 outputs. This matters for
* prefill: the first prototype reloaded/converter A once per 16 output cols. */
__global__ static void w4a16_matmul(float *y,const float *x,const uint8_t *w,
@@ -1748,6 +1756,50 @@ extern "C" int coli_cuda_matmul_mxfp4(float *y, const float *x,
return ok;
}
+/* Reuse one weight/scale staging allocation across the three projections.
+ * Default-stream copies are ordered after the previous projection's reads. */
+static int mxfp4_project(float *y, const float *x, uint8_t *dw, uint8_t *ds,
+ const uint8_t *w, const uint8_t *sc, int S, int I, int O) {
+ size_t rb = ((size_t)I + 1) / 2, ng = ((size_t)I + 31) / 32;
+ if (!cuda_ok(cudaMemcpy(dw, w, (size_t)O * rb, cudaMemcpyHostToDevice), "expert weight upload") ||
+ !cuda_ok(cudaMemcpy(ds, sc, (size_t)O * ng, cudaMemcpyHostToDevice), "expert scale upload")) return 0;
+ quant_matmul<<<dim3(O, S), 256>>>(y, x, dw, reinterpret_cast<const float *>(ds),
+ 7, S, I, O, rb, 32, (int)ng);
+ return cuda_ok(cudaGetLastError(), "MXFP4 expert projection");
+}
+
+extern "C" int coli_cuda_expert_mxfp4(float *y, const float *x,
+ const unsigned char *gate_w, const unsigned char *gate_s,
+ const unsigned char *up_w, const unsigned char *up_s,
+ const unsigned char *down_w, const unsigned char *down_s,
+ int S, int D, int I, float b1, float b2) {
+ if (fault_injected() || !x || !y || !gate_w || !gate_s || !up_w || !up_s ||
+ !down_w || !down_s || S < 1 || S > 65535 || D < 1 || I < 1 ||
+ !(b1 > 0.f) || !(b2 > 0.f) || !std::isfinite(b1) || !std::isfinite(b2)) return 0;
+ DeviceContext *ctx = find_ctx(0);
+ if (!select_ctx(ctx)) return 0;
+ size_t xb = (size_t)S * D * sizeof(float), ib = (size_t)S * I * sizeof(float);
+ if (!reserve(&ctx->x, &ctx->x_cap, xb) || !reserve(&ctx->y, &ctx->y_cap, xb) ||
+ !reserve(&ctx->gate, &ctx->gate_cap, ib) || !reserve(&ctx->up, &ctx->up_cap, ib)) return 0;
+ size_t gw = (size_t)I * (((size_t)D + 1) / 2), dwb = (size_t)D * (((size_t)I + 1) / 2);
+ size_t gs = (size_t)I * (((size_t)D + 31) / 32), dsb = (size_t)D * (((size_t)I + 31) / 32);
+ uint8_t *dw = nullptr, *ds = nullptr;
+ if (!cuda_ok(cudaMalloc(&dw, gw > dwb ? gw : dwb), "expert weight scratch")) return 0;
+ if (!cuda_ok(cudaMalloc(&ds, gs > dsb ? gs : dsb), "expert scale scratch")) { cudaFree(dw); return 0; }
+ int ok = cuda_ok(cudaMemcpy(ctx->x, x, xb, cudaMemcpyHostToDevice), "expert input upload") &&
+ mxfp4_project(ctx->gate, ctx->x, dw, ds, gate_w, gate_s, S, D, I) &&
+ mxfp4_project(ctx->up, ctx->x, dw, ds, up_w, up_s, S, D, I);
+ if (ok) {
+ size_t n = (size_t)S * I;
+ situ_mul<<<(unsigned)((n + 255) / 256), 256>>>(ctx->gate, ctx->up, n, b1, b2);
+ ok = cuda_ok(cudaGetLastError(), "SiTU-GLU launch") &&
+ mxfp4_project(ctx->y, ctx->gate, dw, ds, down_w, down_s, S, I, D) &&
+ cuda_ok(cudaMemcpy(y, ctx->y, xb, cudaMemcpyDeviceToHost), "expert output download");
+ }
+ cudaFree(dw); cudaFree(ds);
+ return ok;
+}
+
extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,
ColiCudaTensor *down, float *y,
const float *x, int S) {
diff --git a/c/backend_cuda.h b/c/backend_cuda.h
index 953e8b39..d2c8b92d 100644
--- a/c/backend_cuda.h
+++ b/c/backend_cuda.h
@@ -115,6 +115,15 @@ COLI_CUDA_DLLEXPORT int coli_cuda_matmul_mxfp4(float *y, const float *x,
const unsigned char *e8s,
int S, int I, int O);
+/* Streaming Kimi expert: down(SiTU(gate(x), up(x))). Weights are MXFP4
+ * host buffers; intermediate activations remain on device. No weight cache.
+ * Returns 0 on failure; callers must accumulate y only after success. */
+COLI_CUDA_DLLEXPORT int coli_cuda_expert_mxfp4(float *y, const float *x,
+ const unsigned char *gate_w, const unsigned char *gate_s,
+ const unsigned char *up_w, const unsigned char *up_s,
+ const unsigned char *down_w, const unsigned char *down_s,
+ int S, int D, int I, float b1, float b2);
+
COLI_CUDA_DLLEXPORT int coli_cuda_matmul(ColiCudaTensor **tensor,
float *y, const float *x,
const void *weights, const float *scales,
diff --git a/c/backend_loader.c b/c/backend_loader.c
index d877553a..a3d4db20 100644
--- a/c/backend_loader.c
+++ b/c/backend_loader.c
@@ -102,6 +102,11 @@ typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const flo
int fmt, int S, int I, int O, int device, int gs);
typedef int (*fn_matmul_mxfp4)(float *y, const float *x, const unsigned char *q4,
const unsigned char *e8s, int S, int I, int O);
+typedef int (*fn_expert_mxfp4)(float *y, const float *x,
+ const unsigned char *gate_w, const unsigned char *gate_s,
+ const unsigned char *up_w, const unsigned char *up_s,
+ const unsigned char *down_w, const unsigned char *down_s,
+ int S, int D, int I, float b1, float b2);
typedef void (*fn_tensor_free)(ColiCudaTensor *tensor);
typedef size_t (*fn_tensor_bytes)(const ColiCudaTensor *tensor);
typedef size_t (*fn_tensor_vram)(const ColiCudaTensor *tensor);
@@ -179,6 +184,7 @@ static struct {
fn_fp8_set_lut fp8_set_lut;
fn_matmul matmul;
fn_matmul_mxfp4 matmul_mxfp4;
+ fn_expert_mxfp4 expert_mxfp4;
fn_tensor_free tensor_free;
fn_tensor_bytes tensor_bytes;
fn_tensor_vram tensor_vram;
@@ -1433,6 +1439,7 @@ static int coli_cuda_load(void){
* nothing by this name, and the wrapper's 0 is the engine's own "fall back
* to CPU" result, so an older DLL still serves GLM and Qwen3.6 (#1405). */
RESOLVE_OPT(matmul_mxfp4, fn_matmul_mxfp4)
+ RESOLVE_OPT(expert_mxfp4, fn_expert_mxfp4)
RESOLVE_OPT(available_device_count, fn_available_device_count) /* qwen36 tier (#1533); older DLLs fall back to device_count */
RESOLVE(tensor_free, fn_tensor_free)
RESOLVE(tensor_bytes, fn_tensor_bytes)
@@ -1656,6 +1663,15 @@ int coli_cuda_matmul_mxfp4(float *y, const float *x, const unsigned char *q4,
return g_cuda.matmul_mxfp4(y, x, q4, e8s, S, I, O);
}
+int coli_cuda_expert_mxfp4(float *y, const float *x,
+ const unsigned char *gate_w, const unsigned char *gate_s,
+ const unsigned char *up_w, const unsigned char *up_s,
+ const unsigned char *down_w, const unsigned char *down_s,
+ int S, int D, int I, float b1, float b2) {
+ if (!g_cuda.available || !g_cuda.expert_mxfp4) return 0;
+ return g_cuda.expert_mxfp4(y, x, gate_w, gate_s, up_w, up_s, down_w, down_s, S, D, I, b1, b2);
+}
+
void coli_cuda_tensor_free(ColiCudaTensor *tensor){
if(g_cuda.available && g_cuda.tensor_free) g_cuda.tensor_free(tensor);
}
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index 5543ad61..9dd73357 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -1612,28 +1612,22 @@ static inline float situf_(float g, float u, float b1, float b2){
}
#ifdef COLI_CUDA
-/* CUDA apply for one expert, decode only (S==1).
- *
- * Same shape as the Vulkan path below and the CPU expert_apply above -- w1/w3,
- * SiTU-GLU on the host, then w2 down -- but stateless: the routed tier streams,
- * so there is nothing resident to keep a device handle for. Weights ride up
- * with the call.
- *
- * Returns 0 with u untouched on ANY failure, so the caller falls through to the
- * disk+CPU path exactly as it does when Vulkan declines. That is the contract
- * vLLM's MXFP4 backends use too -- FlashInfer/AITER when they can, an emulation
- * path when they cannot -- and it is what makes the fast path safe to attempt
- * unconditionally. */
+/* Keep SiTU-GLU and intermediate activations on the GPU when supported.
+ * Older DLLs lack the optional fused entry point, so retain the three-matmul
+ * path as fallback. Accumulate into u only after a complete expert succeeds. */
static int cuda_expert_apply(Model *m, const uint8_t *w1p, const uint8_t *w1s,
const uint8_t *w2p, const uint8_t *w2s,
const uint8_t *w3p, const uint8_t *w3s,
const float *z, float wk,
float *u, float *gate, float *up, float *hz){
Cfg *c=&m->c;
- if(!coli_cuda_matmul_mxfp4(gate,z,w1p,w1s,1,c->latent,c->moe_inter)) return 0;
- if(!coli_cuda_matmul_mxfp4(up, z,w3p,w3s,1,c->latent,c->moe_inter)) return 0;
- for(int i=0;i<c->moe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);
- if(!coli_cuda_matmul_mxfp4(hz,gate,w2p,w2s,1,c->moe_inter,c->latent)) return 0;
+ if(!coli_cuda_expert_mxfp4(hz,z,w1p,w1s,w3p,w3s,w2p,w2s,
+ 1,c->latent,c->moe_inter,c->situ_b1,c->situ_b2)) {
+ if(!coli_cuda_matmul_mxfp4(gate,z,w1p,w1s,1,c->latent,c->moe_inter)) return 0;
+ if(!coli_cuda_matmul_mxfp4(up, z,w3p,w3s,1,c->latent,c->moe_inter)) return 0;
+ for(int i=0;i<c->moe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);
+ if(!coli_cuda_matmul_mxfp4(hz,gate,w2p,w2s,1,c->moe_inter,c->latent)) return 0;
+ }
for(int i=0;i<c->latent;i++) u[i]+=wk*hz[i];
return 1;
}
diff --git a/c/tests/test_kimi_cuda_expert.c b/c/tests/test_kimi_cuda_expert.c
new file mode 100644
index 00000000..6815e0f5
--- /dev/null
+++ b/c/tests/test_kimi_cuda_expert.c
@@ -0,0 +1,39 @@
+/* Exercise the actual engine dispatch without a GPU, including an older DLL. */
+#define main kimi_main_unused
+#include "../kimi_k3.c"
+#undef main
+
+static int fused_ok, legacy_ok, fused_calls, legacy_calls;
+int coli_cuda_init(const int *devices, int n) { return 1; }
+int coli_cuda_expert_mxfp4(float *y, const float *x,
+ const unsigned char *gw, const unsigned char *gs,
+ const unsigned char *uw, const unsigned char *us,
+ const unsigned char *dw, const unsigned char *ds,
+ int S, int D, int I, float b1, float b2) {
+ fused_calls++;
+ for (int i = 0; i < D; i++) y[i] = fused_ok ? 2.f : -999.f;
+ return fused_ok;
+}
+int coli_cuda_matmul_mxfp4(float *y, const float *x, const unsigned char *q,
+ const unsigned char *sc, int S, int I, int O) {
+ legacy_calls++;
+ for (int i = 0; i < O; i++) y[i] = 3.f;
+ return legacy_ok;
+}
+int main(void) {
+ Model m = {0}; m.c.latent = 3; m.c.moe_inter = 5;
+ m.c.situ_b1 = 2.f; m.c.situ_b2 = 3.f;
+ uint8_t w[16] = {0}, sc[5] = {0};
+ float x[3] = {1,2,3}, out[3] = {4,4,4}, gate[5], up[5], hz[3];
+ fused_ok = 1;
+ if (!cuda_expert_apply(&m,w,sc,w,sc,w,sc,x,0.5f,out,gate,up,hz) ||
+ fused_calls != 1 || legacy_calls || out[0] != 5.f || out[2] != 5.f) return 1;
+ fused_ok = 0; legacy_ok = 1;
+ if (!cuda_expert_apply(&m,w,sc,w,sc,w,sc,x,0.5f,out,gate,up,hz) ||
+ legacy_calls != 3 || out[0] != 6.5f || out[2] != 6.5f) return 1;
+ legacy_ok = 0;
+ if (cuda_expert_apply(&m,w,sc,w,sc,w,sc,x,0.5f,out,gate,up,hz) ||
+ out[0] != 6.5f || out[2] != 6.5f) return 1;
+ puts("ok Kimi CUDA fused dispatch, legacy fallback, failed expert does not accumulate");
+ return 0;
+}
diff --git a/c/tests/test_mxfp4_expert_cuda.cu b/c/tests/test_mxfp4_expert_cuda.cu
new file mode 100644
index 00000000..7d9e67e2
--- /dev/null
+++ b/c/tests/test_mxfp4_expert_cuda.cu
@@ -0,0 +1,55 @@
+/* Streaming SiTU-GLU expert vs the CPU MXFP4 decoder, including odd widths. */
+#include <cmath>
+#include <cstdio>
+#include <vector>
+#if defined(__HIPCC__)
+#include "../backend_gpu_compat.h"
+#else
+#include <cuda_runtime.h>
+#endif
+#include "../backend_cuda.h"
+extern "C" void mxfp4_ref(float *, const float *, const unsigned char *, const unsigned char *, int, int, int);
+
+static int check_case(int S, int D, int I, float b1, float b2) {
+ std::vector<unsigned char> gw((size_t)I * ((D + 1) / 2)), uw(gw.size()), dw((size_t)D * ((I + 1) / 2));
+ std::vector<unsigned char> gs((size_t)I * ((D + 31) / 32)), us(gs.size()), ds((size_t)D * ((I + 31) / 32));
+ std::vector<float> x((size_t)S * D), g((size_t)S * I), u(g.size()), want(x.size()), got(x.size());
+ for (size_t j = 0; j < gw.size(); j++) { gw[j] = (unsigned char)(j * 13 + 2); uw[j] = (unsigned char)(j * 7 + 5); }
+ for (size_t j = 0; j < dw.size(); j++) dw[j] = (unsigned char)(j * 11 + 3);
+ for (size_t j = 0; j < gs.size(); j++) { gs[j] = 121 + j % 4; us[j] = 122 + j % 3; }
+ for (size_t j = 0; j < ds.size(); j++) ds[j] = 123 + j % 4;
+ for (size_t j = 0; j < x.size(); j++) x[j] = (int(j % 17) - 8) * 0.125f;
+ for (int pass = 0; pass < 2; pass++) {
+ mxfp4_ref(g.data(), x.data(), gw.data(), gs.data(), S, D, I);
+ mxfp4_ref(u.data(), x.data(), uw.data(), us.data(), S, D, I);
+ for (size_t j = 0; j < g.size(); j++)
+ g[j] = b1 * tanhf(g[j] / b1) * (1.f / (1.f + expf(-g[j]))) * b2 * tanhf(u[j] / b2);
+ mxfp4_ref(want.data(), g.data(), dw.data(), ds.data(), S, I, D);
+ if (!coli_cuda_expert_mxfp4(got.data(), x.data(), gw.data(), gs.data(), uw.data(), us.data(),
+ dw.data(), ds.data(), S, D, I, b1, b2)) return 1;
+ for (size_t j = 0; j < got.size(); j++) {
+ if (!std::isfinite(got[j]) || fabsf(got[j] - want[j]) > 2e-5f + 2e-4f * fabsf(want[j])) {
+ printf("FAIL S=%d D=%d I=%d row-element=%zu want=%g got=%g\n", S, D, I, j, want[j], got[j]);
+ return 1;
+ }
+ }
+ /* Reused host addresses now contain different expert bytes. */
+ for (size_t j = 0; j < gw.size(); j++) gw[j] ^= 0x88;
+ for (size_t j = 0; j < ds.size(); j++) ds[j]++;
+ }
+ got[0] = 123.f;
+ if (coli_cuda_expert_mxfp4(got.data(), x.data(), gw.data(), gs.data(), uw.data(), us.data(),
+ dw.data(), ds.data(), S, D, I, 0.f, b2) || got[0] != 123.f) return 1;
+ printf("ok SiTU expert S=%d D=%d I=%d b1=%g b2=%g\n", S, D, I, b1, b2);
+ return 0;
+}
+int main() {
+ int n = 0, device = 0;
+ if (cudaGetDeviceCount(&n) != cudaSuccess || !n) { puts("SKIP no CUDA device"); return 0; }
+ if (!coli_cuda_init(&device, 1)) return 1;
+ int failed = check_case(1, 64, 128, 1.5f, 2.5f) |
+ check_case(3, 33, 65, 4.f, 3.f) |
+ check_case(2, 129, 31, 0.5f, 0.75f);
+ coli_cuda_shutdown();
+ return failed;
+}
diff --git a/c/tools/clean.py b/c/tools/clean.py
index c309212a..ce08c299 100644
--- a/c/tools/clean.py
+++ b/c/tools/clean.py
@@ -27,6 +27,7 @@ FILES = [
"iobench", "iobench.exe",
"backend_cuda.o", "backend_loader.o",
"backend_cuda_test", "backend_cuda_test.exe",
+ "mxfp4_expert_cuda_test", "mxfp4_expert_cuda_test.exe",
"backend_cuda_bench", "backend_cuda_bench.exe",
"backend_metal.o", "backend_metal_test",
"coli_cuda.dll", "coli_cuda.lib", "coli_cuda.exp",