Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Decode MXFP4 (fmt=7) on CUDA, so Kimi K3 has a CUDA expert tier

ZacharyZcRdb0c80f0d368
diff --git a/c/.gitignore b/c/.gitignore
index 356fdfc7..64e5d0ec 100644
--- a/c/.gitignore
+++ b/c/.gitignore
@@ -13,6 +13,8 @@ tools/librans_c.dylib
tools/rans_c.dll
tests/fuzz_rans
tests/bench_omp_grain
+tests/mxfp4_ref.o
+mxfp4_cuda_test
tests/*.dSYM/
tests/test_dsa_select
tests/test_i4_grouped
diff --git a/c/Makefile b/c/Makefile
index e630eb54..4a349411 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -660,12 +660,18 @@ rans: $(RANSLIB)
$(RANSLIB): tools/rans_ctypes.c rans.h
$(CC) $(CFLAGS) -fPIC -shared $< -o $@ $(LDFLAGS)
-cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu
+cuda-test: backend_cuda.cu backend_cuda.h backend_gpu_compat.h tests/test_backend_cuda.cu tests/test_ragged_attention.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.c
@command -v "$(GPUCC)" >/dev/null 2>&1 || { echo "$(GPUCC_NAME) not found" >&2; exit 1; }
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) $(ANS_NVCC_LIBS)
./backend_cuda_test$(EXE)
"$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_ragged_attention.cu -o ragged_attention_test$(EXE)
./ragged_attention_test$(EXE)
+ # MXFP4 (fmt=7) against the CPU decoder in quant.h. The reference is built
+ # as C on purpose: quant.h uses _Thread_local, which nvcc's C++ front end
+ # rejects, and re-implementing it for the test would defeat the comparison.
+ $(CC) $(CFLAGS) -c tests/mxfp4_ref.c -o tests/mxfp4_ref.o
+ "$(GPUCC)" $(GPUFLAGS) backend_cuda.cu tests/test_mxfp4_cuda.cu tests/mxfp4_ref.o -o mxfp4_cuda_test$(EXE)
+ ./mxfp4_cuda_test$(EXE)
# convenience alias: kernel correctness on AMD (same test, hipcc toolchain)
hip-test:
@@ -700,12 +706,8 @@ NOCUDA_CFLAGS = $(filter-out -DCOLI_CUDA -DCOLI_ANS,$(CFLAGS))
NOCUDA_LDFLAGS = $(filter-out -lcudart -lstdc++ -lcuda -L$(CUDA_HOME)/lib64 \
-Wl$(comma)-rpath$(comma)$(CUDA_HOME)/lib64,$(LDFLAGS))
-kimi_k3$(EXE): kimi_k3.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h $(VK_OBJ) $(VK_SPV)
-ifeq ($(CUDA),1)
- @echo "*** kimi_k3 has no CUDA backend (#783): building without it."
- @echo "*** On NVIDIA, Kimi K3 runs on the Vulkan path β€” build VULKAN=1 and set COLI_VULKAN=1."
-endif
- $(CC) $(NOCUDA_CFLAGS) kimi_k3.c $(VK_OBJ) -o kimi_k3$(EXE) $(NOCUDA_LDFLAGS)
+kimi_k3$(EXE): kimi_k3.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h omp_tune.h route_trace.h backend_cuda.h $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV)
+ $(CC) $(CFLAGS) kimi_k3.c $(CUDA_OBJ) $(VK_OBJ) -o kimi_k3$(EXE) $(LDFLAGS)
# Use a baseline that matches the compiler target. macOS already targets a
# portable baseline when ARCH is empty; forcing the x86 value there breaks
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 50631e1a..5a08e06e 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -165,6 +165,7 @@ __host__ __device__ static size_t row_bytes(int fmt, int I) {
if (fmt == 2 || fmt == 4) return (size_t)(I + 1) / 2; /* fmt=4: same packed int4 */
if (fmt == 3) return (size_t)(I + 3) / 4;
if (fmt == 4) return (size_t)(I + 1) / 2; /* grouped int4: nibbles like fmt 2 */
+ if (fmt == 7) return (size_t)(I + 1) / 2; /* MXFP4: e2m1 nibbles, 2 per byte */
if (fmt == 6) return (size_t)(((int64_t)I + COLI_E8_QK - 1) / COLI_E8_QK) * COLI_E8_BBYTES;
if (fmt == 8) return (size_t)I; /* fp8-e4m3: raw bytes, layout of fmt=1 */
return 0;
@@ -215,6 +216,45 @@ __device__ __forceinline__ void e8_expand_sub_dev(const uint8_t *blk, int ib, fl
}
}
+/* ---- MXFP4 (OCP microscaling FP4), fmt=7 -----------------------------------
+ * Same layout the CPU path decodes in quant.h's matmul_mxfp4, and the same two
+ * tricks, so the two agree bit for bit:
+ *
+ * packed [O, I/2] u8 β€” e2m1 nibbles, LOW nibble = even column, bit3 = sign,
+ * bits 0..2 index {0,.5,1,1.5,2,3,4,6}
+ * scales [O, I/32] u8 β€” ue8m0 exponent per 32-column group, w = v * 2^(s-127)
+ *
+ * The exponent is decoded as a bit pattern rather than exp2f: (uint32)s << 23
+ * reinterpreted as float IS 2^(s-127) for s in [1,254], and reproduces the CPU
+ * path's documented edge behaviour exactly -- s=0 gives +0 and s=255 gives +inf
+ * on both sides. Using exp2f here would agree for the normal range and diverge
+ * at the ends, which is precisely where a silent mismatch would hide.
+ *
+ * The LUT holds DOUBLED values so every entry is an exact small integer; the
+ * compensating 0.5f rides along in mx4_scale_dev, as it does on the CPU. */
+/* Decoded arithmetically rather than from a __constant__ table: a file-scope
+ * __constant__ array with static linkage is initialised per translation unit,
+ * and this kernel is also compiled into the HIP build and the DLL, where that
+ * silently yields garbage. The magnitude is 2^(exp-1) * 0.5 for exp in 1..3 and
+ * 0 for exp 0, which is exactly the OCP e2m1 table {0,.5,1,1.5,2,3,4,6}. */
+__device__ static inline float mx4_decode(int n) {
+ int mant = n & 1, exp = (n >> 1) & 3;
+ float mag = exp ? ldexpf(1.0f + 0.5f * (float)mant, exp - 1) : 0.5f * (float)mant;
+ return (n & 8) ? -mag : mag;
+}
+
+__device__ static inline float mx4_scale_dev(uint8_t s) {
+ union { uint32_t u; float f; } b;
+ b.u = static_cast<uint32_t>(s) << 23;
+ return b.f;
+}
+
+/* e2m1 nibble at column i of a packed row. */
+__device__ static inline float mx4_weight_at(const uint8_t *q, int i) {
+ uint8_t v = q[i >> 1];
+ return mx4_decode((i & 1) ? (v >> 4) : (v & 15));
+}
+
__device__ static float weight_at(const void *weights, int fmt, size_t row, int i) {
const uint8_t *base = static_cast<const uint8_t *>(weights) + row;
if (fmt == 0) return reinterpret_cast<const float *>(base)[i];
@@ -265,6 +305,18 @@ __global__ static void quant_matmul(float *y, const float *x, const void *weight
int off = sb*COLI_E8_SUB, n = I-off < COLI_E8_SUB ? I-off : COLI_E8_SUB;
for (int k=0;k<n;k++) sum += xs[off+k]*w[k];
}
+ } else if (fmt == 7) {
+ /* MXFP4: one ue8m0 exponent per 32 columns. Threads stride over columns
+ * and pick up the group exponent as they cross a boundary -- the same
+ * accumulation order as the CPU scalar path, so a mismatch means the
+ * decode is wrong, not the summation. */
+ const uint8_t *wrow = static_cast<const uint8_t *>(weights) + row;
+ const uint8_t *scl = reinterpret_cast<const uint8_t *>(scales) + (size_t)o * ng;
+ for (int i = threadIdx.x; i < I; i += blockDim.x) {
+ int g = i >> 5;
+ if (g >= ng) g = ng - 1;
+ sum += xs[i] * mx4_weight_at(wrow, i) * mx4_scale_dev(scl[g]);
+ }
} else if (fmt == 4) {
/* Grouped int4: one f32 scale per gs elements along I (ng groups per row).
* Scale layout: scales[o*ng + g]. Each thread strides through I, applying
@@ -300,7 +352,13 @@ __global__ static void quant_matmul(float *y, const float *x, const void *weight
__syncthreads();
}
if (!threadIdx.x)
- y[(size_t)s * O + o] = (fmt && fmt != 4 && fmt != 6 && fmt != 8) ? partial[0] * scales[o] : partial[0];
+ /* fmt 4/6/7/8 already applied their scaling inside the loop: 4 and 7 are
+ * per-group (one scale per gs / per 32 columns), 6 carries it in the
+ * block header, 8 reads a per-128 block scale alongside the weights.
+ * Only the per-row formats get the trailing multiply --
+ * and for fmt=7 `scales` points at ue8m0 BYTES, so reading it as float
+ * here does not merely double-scale, it reads garbage. */
+ y[(size_t)s * O + o] = (fmt && fmt != 4 && fmt != 6 && fmt != 7 && fmt != 8) ? partial[0] * scales[o] : partial[0];
}
/* fmt=6 activation rotation, y = Q^T x for Q = D*H/sqrt(n) (#452). One block per
@@ -1316,6 +1374,47 @@ extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor,
return 1;
}
+/* MXFP4 matmul, stateless. Separate from coli_cuda_matmul on purpose: that one
+ * takes scales as const float* and caches an uploaded tensor, while MXFP4
+ * scales are ue8m0 BYTES -- passing them through the float* parameter would
+ * compile and silently reinterpret the buffer. Kimi K3's routed experts stream
+ * (a fill-once tier at decode), so there is nothing to cache here anyway; the
+ * weights go up with the call.
+ *
+ * Returns 0 and leaves y untouched on any failure, which is the contract the
+ * engine's GPU paths already use to fall back to CPU. */
+extern "C" int coli_cuda_matmul_mxfp4(float *y, const float *x,
+ const uint8_t *q4, const uint8_t *e8s,
+ int S, int I, int O) {
+ if (fault_injected()) return 0;
+ if (S < 1 || I < 1 || O < 1 || !y || !x || !q4 || !e8s) return 0;
+ DeviceContext *ctx = find_ctx(0);
+ if (!select_ctx(ctx)) return 0;
+
+ size_t rb = (size_t)(I + 1) / 2, ng = (size_t)(I + 31) / 32;
+ size_t wb = (size_t)O * rb, sb = (size_t)O * ng;
+ size_t xb = (size_t)S * I * sizeof(float), yb = (size_t)S * O * sizeof(float);
+
+ uint8_t *dw = nullptr, *ds = nullptr;
+ if (!cuda_ok(cudaMalloc(&dw, wb), "mxfp4 weight alloc")) return 0;
+ if (!cuda_ok(cudaMalloc(&ds, sb), "mxfp4 scale alloc")) { cudaFree(dw); return 0; }
+
+ int ok = reserve(&ctx->x, &ctx->x_cap, xb) && reserve(&ctx->y, &ctx->y_cap, yb) &&
+ cuda_ok(cudaMemcpy(dw, q4, wb, cudaMemcpyHostToDevice), "mxfp4 weight upload") &&
+ cuda_ok(cudaMemcpy(ds, e8s, sb, cudaMemcpyHostToDevice), "mxfp4 scale upload") &&
+ cuda_ok(cudaMemcpy(ctx->x, x, xb, cudaMemcpyHostToDevice), "mxfp4 input upload");
+ if (ok) {
+ dim3 grid((unsigned)O, (unsigned)S);
+ quant_matmul<<<grid, 256>>>(ctx->y, ctx->x, dw, reinterpret_cast<const float *>(ds),
+ 7, S, I, O, rb, 32, (int)ng);
+ ok = cuda_ok(cudaGetLastError(), "mxfp4 launch") &&
+ cuda_ok(cudaMemcpy(y, ctx->y, yb, cudaMemcpyDeviceToHost), "mxfp4 output download");
+ }
+ cudaFree(dw);
+ cudaFree(ds);
+ return ok;
+}
+
extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,
ColiCudaTensor *down, float *y,
const float *x, int S) {
diff --git a/c/backend_cuda.h b/c/backend_cuda.h
index 45453987..d614aa6b 100644
--- a/c/backend_cuda.h
+++ b/c/backend_cuda.h
@@ -73,6 +73,16 @@ COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_compressed(ColiCudaTensor **tens
* The first successful call uploads W and its scales; later calls reuse it.
* Returns 1 on success and 0 when CUDA is not initialized or the format is invalid.
*/
+/* y[S,O] = x[S,I] @ dequant_mxfp4(W[O,I])^T, fmt=7 (OCP microscaling FP4).
+ * q4 is [O, ceil(I/2)] e2m1 nibbles, e8s is [O, ceil(I/32)] ue8m0 exponents --
+ * BYTES, not floats, which is why this is not folded into coli_cuda_matmul.
+ * Stateless: weights are uploaded per call, matching the streaming expert tier
+ * Kimi K3 uses. Returns 1 on success, 0 (y untouched) to fall back to CPU. */
+COLI_CUDA_DLLEXPORT int coli_cuda_matmul_mxfp4(float *y, const float *x,
+ const unsigned char *q4,
+ const unsigned char *e8s,
+ int S, int I, int O);
+
COLI_CUDA_DLLEXPORT int coli_cuda_matmul(ColiCudaTensor **tensor,
float *y, const float *x,
const void *weights, const float *scales,
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index d7e1ad34..312e859f 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -80,6 +80,9 @@
#include "st.h"
#include "tok.h"
#include "quant.h"
+#ifdef COLI_CUDA
+#include "backend_cuda.h"
+#endif
#include "omp_tune.h"
#include "route_trace.h"
#include "kv_prefix.h" /* KV prefix reuse (shared) */
@@ -287,6 +290,9 @@ static float w_rowdot(const W *w, int r, const float *x){
#define QCHUNK 1024 /* rows per load-quantize pass */
static int g_bits_env=0; /* K3_BITS explicitly set: enables the
* int8-container -> int4 load downcast */
+#ifdef COLI_CUDA
+static int g_k3_cuda=0; /* K3_CUDA=1: MXFP4 routed experts on CUDA at decode */
+#endif
static int g_k3_direct=-1; /* K3_DIRECT: O_DIRECT expert reads */
static int g_k3_idot=1; /* K3_IDOT: int8-activation expert matmuls */
static int g_k3_pipe=1; /* K3_PIPE: overlap loads with compute */
@@ -498,6 +504,16 @@ static void model_init(Model *m, const char *snap, int n_layers_env){
snprintf(m->pfx,sizeof(m->pfx),"language_model.");
if((c->n_layers+c->res_bs-1)/c->res_bs+1>16){ fprintf(stderr,"attn_res: too many blocks\n"); exit(1); }
g_bits_env = getenv("K3_BITS")!=NULL;
+#ifdef COLI_CUDA
+ g_k3_cuda = getenv("K3_CUDA") ? atoi(getenv("K3_CUDA")) : 0;
+ if(g_k3_cuda){
+ int dev0 = 0;
+ if(!coli_cuda_init(&dev0, 1)){
+ fprintf(stderr,"[K3-CUDA] device unavailable -- experts stay on CPU\n");
+ g_k3_cuda = 0;
+ } else fprintf(stderr,"[K3-CUDA] MXFP4 routed experts on device 0 (decode only)\n");
+ }
+#endif
g_k3_direct = getenv("K3_DIRECT")?atoi(getenv("K3_DIRECT")):1;
g_k3_idot = getenv("K3_IDOT")?atoi(getenv("K3_IDOT")):1;
g_k3_pipe = getenv("K3_PIPE")?atoi(getenv("K3_PIPE")):1;
@@ -866,6 +882,34 @@ static inline float situf_(float g, float u, float b1, float b2){
return b1*tanhf(g/b1)*sigmoidf_(g) * b2*tanhf(u/b2);
}
+#ifdef COLI_CUDA
+/* CUDA apply for one expert, decode only (S==1).
+ *
+ * Same shape as the Vulkan path below and the CPU expert_apply above -- w1/w3,
+ * SiTU-GLU on the host, then w2 down -- but stateless: the routed tier streams,
+ * so there is nothing resident to keep a device handle for. Weights ride up
+ * with the call.
+ *
+ * Returns 0 with u untouched on ANY failure, so the caller falls through to the
+ * disk+CPU path exactly as it does when Vulkan declines. That is the contract
+ * vLLM's MXFP4 backends use too -- FlashInfer/AITER when they can, an emulation
+ * path when they cannot -- and it is what makes the fast path safe to attempt
+ * unconditionally. */
+static int cuda_expert_apply(Model *m, const uint8_t *w1p, const uint8_t *w1s,
+ const uint8_t *w2p, const uint8_t *w2s,
+ const uint8_t *w3p, const uint8_t *w3s,
+ const float *z, float wk,
+ float *u, float *gate, float *up, float *hz){
+ Cfg *c=&m->c;
+ if(!coli_cuda_matmul_mxfp4(gate,z,w1p,w1s,1,c->latent,c->moe_inter)) return 0;
+ if(!coli_cuda_matmul_mxfp4(up, z,w3p,w3s,1,c->latent,c->moe_inter)) return 0;
+ for(int i=0;i<c->moe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);
+ if(!coli_cuda_matmul_mxfp4(hz,gate,w2p,w2s,1,c->moe_inter,c->latent)) return 0;
+ for(int i=0;i<c->latent;i++) u[i]+=wk*hz[i];
+ return 1;
+}
+#endif
+
/* u += wk * E(z) for one loaded expert slot (SiTU-GLU in the latent).
* gate/up are [moe_inter] scratch, hz is [latent] scratch. */
static void expert_apply(Model *m, Slot *s, const float *z, float wk,
@@ -873,6 +917,11 @@ static void expert_apply(Model *m, Slot *s, const float *z, float wk,
Cfg *c=&m->c;
uint8_t *w1p=s->buf, *w1s=w1p+m->e_w1p, *w2p=w1s+m->e_w1s, *w2s=w2p+m->e_w2p,
*w3p=w2s+m->e_w2s, *w3s=w3p+m->e_w1p;
+#ifdef COLI_CUDA
+ /* Opt-in (K3_CUDA=1), decode only: at S>1 the CPU kernels amortise across
+ * the batch and the per-call upload would not pay for itself. */
+ if(g_k3_cuda && cuda_expert_apply(m,w1p,w1s,w2p,w2s,w3p,w3s,z,wk,u,gate,up,hz)) return;
+#endif
void (*mm)(float*,const float*,const uint8_t*,const uint8_t*,int,int,int)
= g_k3_idot ? matmul_mxfp4_i8 : matmul_mxfp4;
mm(gate,z,w1p,w1s,1,c->latent,c->moe_inter);
diff --git a/c/tests/mxfp4_ref.c b/c/tests/mxfp4_ref.c
new file mode 100644
index 00000000..a4703db1
--- /dev/null
+++ b/c/tests/mxfp4_ref.c
@@ -0,0 +1,12 @@
+/* CPU MXFP4 reference for tests/test_mxfp4_cuda.cu.
+ *
+ * quant.h is C: it declares `static _Thread_local QScratch g_qscratch`, which
+ * nvcc's C++ front end rejects. Compiling the reference here as C and exposing
+ * one symbol keeps the test comparing against the SAME code the engine runs,
+ * rather than a C++-friendly re-implementation that could drift from it. */
+#include "../quant.h"
+
+void mxfp4_ref(float *y, const float *x, const unsigned char *q4,
+ const unsigned char *e8s, int S, int I, int O) {
+ matmul_mxfp4(y, x, q4, e8s, S, I, O);
+}
diff --git a/c/tests/test_mxfp4_cuda.cu b/c/tests/test_mxfp4_cuda.cu
new file mode 100644
index 00000000..3dada8ad
--- /dev/null
+++ b/c/tests/test_mxfp4_cuda.cu
@@ -0,0 +1,158 @@
+/* MXFP4 (fmt=7) on CUDA, checked against the CPU decoder that already ships.
+ *
+ * Kimi K3's routed experts are QAT in MXFP4 and passed through un-re-encoded,
+ * so fmt=7 is the format its expert tier runs on. Until now only the Vulkan
+ * shader could decode it; the CUDA backend understood fmt 0/1/2/3/4/6 and
+ * nothing else, so a CUDA host had no expert tier for that engine at all.
+ *
+ * The reference is quant.h's matmul_mxfp4 -- the CPU path the engine already
+ * uses and trusts. That makes this a differential test rather than a
+ * self-consistent one: if both sides are wrong in the same way the test says
+ * nothing, but a decode that disagrees with what the engine already computes
+ * is caught immediately, which is the failure that matters when adding a
+ * second implementation of an existing format.
+ *
+ * Tolerance is 1e-4 relative. It is NOT bit-exactness: the two accumulate in a
+ * different order (CPU strides columns serially, CUDA strides them across
+ * threads and reduces), so the float sums legitimately differ in the last
+ * bits. Decode errors are not subtle at this scale -- a wrong nibble, a
+ * swapped parity or a misread exponent moves a result by whole factors, not by
+ * an ulp.
+ *
+ * make -C c cuda-test CUDA_ARCH=native (runs this among the CUDA tests)
+ */
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <cmath>
+#include <cuda_runtime.h>
+
+/* quant.h is C (it uses _Thread_local, which nvcc's C++ front end rejects), so
+ * the reference is compiled separately as C and reached through this one
+ * declaration -- see tests/mxfp4_ref.c. */
+extern "C" void mxfp4_ref(float *y, const float *x, const unsigned char *q4,
+ const unsigned char *e8s, int S, int I, int O);
+#include "../backend_cuda.h"
+
+static int fails;
+
+#define CK(call) \
+ do { \
+ cudaError_t e_ = (call); \
+ if (e_ != cudaSuccess) { \
+ printf(" FAIL %s: %s\n", #call, cudaGetErrorString(e_)); \
+ return 1; \
+ } \
+ } while (0)
+
+static uint64_t rng_state = 0x9E3779B97F4A7C15ull;
+static uint32_t rnd(void) {
+ rng_state ^= rng_state << 13;
+ rng_state ^= rng_state >> 7;
+ rng_state ^= rng_state << 17;
+ return (uint32_t)(rng_state >> 32);
+}
+
+/* One case: random MXFP4 weights, random activations, CPU vs CUDA. */
+static void one_case(const char *what, int S, int I, int O, int fixed_exp) {
+ int rb = (I + 1) / 2, ng = (I + 31) / 32;
+ uint8_t *q4 = (uint8_t *)malloc((size_t)O * rb);
+ uint8_t *e8 = (uint8_t *)malloc((size_t)O * ng);
+ float *x = (float *)malloc(sizeof(float) * S * I);
+ float *y_cpu = (float *)calloc((size_t)S * O, sizeof(float));
+ float *y_gpu = (float *)calloc((size_t)S * O, sizeof(float));
+ if (!q4 || !e8 || !x || !y_cpu || !y_gpu) { printf(" FAIL oom\n"); fails++; return; }
+
+ for (int i = 0; i < O * rb; i++) q4[i] = (uint8_t)(rnd() & 0xFF);
+ for (int i = 0; i < O * ng; i++)
+ /* fixed_exp < 0 keeps exponents in the normal range; a fixed value
+ * pins the edge cases the CPU comment documents (0 -> +0, 255 -> inf). */
+ e8[i] = fixed_exp >= 0 ? (uint8_t)fixed_exp : (uint8_t)(110 + (rnd() % 30));
+ for (int i = 0; i < S * I; i++) x[i] = ((float)(rnd() % 2001) - 1000.0f) / 1000.0f;
+
+ mxfp4_ref(y_cpu, x, q4, e8, S, I, O);
+
+ if (!coli_cuda_matmul_mxfp4(y_gpu, x, q4, e8, S, I, O)) {
+ printf(" FAIL %-28s CUDA path refused the call\n", what);
+ fails++;
+ goto done;
+ }
+
+ {
+ double worst = 0.0;
+ int bad = 0;
+ for (int i = 0; i < S * O; i++) {
+ float a = y_cpu[i], b = y_gpu[i];
+ if (isinf(a) || isinf(b)) { /* exponent 255: both must agree it is inf */
+ if (isinf(a) != isinf(b) || (isinf(a) && ((a > 0) != (b > 0)))) bad++;
+ continue;
+ }
+ double den = fabs(a) > 1e-6 ? fabs(a) : 1e-6;
+ double rel = fabs((double)a - (double)b) / den;
+ if (rel > worst) worst = rel;
+ if (rel > 1e-4) bad++;
+ }
+ if (bad) {
+ printf(" FAIL %-28s %d/%d elements differ, worst rel %.3e\n", what, bad, S * O, worst);
+ fails++;
+ } else {
+ printf(" ok %-28s S=%-3d I=%-5d O=%-4d worst rel %.2e\n", what, S, I, O, worst);
+ }
+ }
+done:
+ free(q4); free(e8); free(x); free(y_cpu); free(y_gpu);
+}
+
+/* Every e2m1 code, decoded in isolation: one column set to code c, x = 1.0,
+ * so the result IS the decoded value times the group scale. */
+static void all_codes(void) {
+ static const float want[16] = {0.f, .5f, 1.f, 1.5f, 2.f, 3.f, 4.f, 6.f,
+ -0.f, -.5f, -1.f, -1.5f, -2.f, -3.f, -4.f, -6.f};
+ int I = 32, O = 16, rb = I / 2, ng = 1;
+ uint8_t q4[16 * 16], e8[16];
+ float x[32], y_cpu[16], y_gpu[16];
+ memset(q4, 0, sizeof q4);
+ for (int o = 0; o < O; o++) { q4[o * rb] = (uint8_t)o; e8[o] = 127; } /* 2^0 = 1 */
+ for (int i = 0; i < I; i++) x[i] = (i == 0) ? 1.0f : 0.0f;
+
+ mxfp4_ref(y_cpu, x, q4, e8, 1, I, O);
+ if (!coli_cuda_matmul_mxfp4(y_gpu, x, q4, e8, 1, I, O)) {
+ printf(" FAIL all-16-e2m1-codes CUDA refused\n");
+ fails++;
+ return;
+ }
+ int bad = 0;
+ for (int o = 0; o < O; o++)
+ if (y_gpu[o] != want[o] || y_cpu[o] != want[o]) {
+ printf(" FAIL code %2d: want %.2f cpu %.2f gpu %.2f\n", o, want[o], y_cpu[o], y_gpu[o]);
+ bad++;
+ }
+ if (bad) fails++;
+ else printf(" ok all 16 e2m1 codes decode exactly (cpu == gpu == spec)\n");
+}
+
+int main(void) {
+ int ndev = 0;
+ if (cudaGetDeviceCount(&ndev) != cudaSuccess || ndev < 1) {
+ printf("test_mxfp4_cuda: no CUDA device, skipping\n");
+ return 0;
+ }
+ int dev0 = 0;
+ if (!coli_cuda_init(&dev0, 1)) {
+ printf("test_mxfp4_cuda: coli_cuda_init failed, skipping\n");
+ return 0;
+ }
+
+ all_codes();
+ one_case("decode + matmul", 1, 64, 32, -1);
+ one_case("multi-row batch", 4, 128, 64, -1);
+ one_case("wide rows (many groups)", 1, 2048, 16, -1);
+ one_case("non-multiple-of-32 columns", 1, 96, 8, -1); /* 3 groups exactly */
+ one_case("tail group (I%32 != 0)", 1, 80, 8, -1); /* 2.5 groups */
+ one_case("exponent 0 -> +0", 1, 64, 16, 0);
+ one_case("exponent 255 -> inf", 1, 64, 16, 255);
+ one_case("exponent 127 -> unit scale", 1, 64, 16, 127);
+
+ printf(fails ? "test_mxfp4_cuda: %d failure(s)\n" : "test_mxfp4_cuda: ok\n", fails);
+ return fails != 0;
+}