Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

cuda: fmt=6 (E8/IQ3) decode, so E8 containers can reach VRAM

ZacharyZcRce25594bde7b
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index 71355c25..5a739cd9 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -87,15 +87,61 @@ static int select_ctx(DeviceContext *ctx) {
return 1;
}
+/* fmt=6 (E8/IQ3) geometry, mirroring quant.h. A super-block packs 256 weights
+ * into 98 bytes: 64 codebook indices, 8 words of (4x7 signs + 4-bit sub-scale),
+ * and one fp16 super-scale. Scales live INSIDE the block, so fmt=6 tensors carry
+ * no separate scale array (#452). */
+#define COLI_E8_QK 256
+#define COLI_E8_SUB 32
+#define COLI_E8_BBYTES 98
+
__host__ __device__ static size_t row_bytes(int fmt, int I) {
if (fmt == 0) return (size_t)I * sizeof(float);
if (fmt == 1) return (size_t)I;
if (fmt == 2 || fmt == 4) return (size_t)(I + 1) / 2; /* fmt=4: same packed int4 */
if (fmt == 3) return (size_t)(I + 3) / 4;
if (fmt == 4) return (size_t)(I + 1) / 2; /* grouped int4: nibbles like fmt 2 */
+ if (fmt == 6) return (size_t)(((int64_t)I + COLI_E8_QK - 1) / COLI_E8_QK) * COLI_E8_BBYTES;
return 0;
}
+/* The E8 codebook, uploaded once per device from quant.h's e8_grid so the table
+ * has a single source of truth and cannot drift from the CPU decoder. */
+__constant__ uint8_t c_e8_grid[256][4];
+
+/* A super-block is 98 bytes, so nothing inside it is guaranteed 4- or 2-byte
+ * aligned: assemble the words byte-wise instead of dereferencing. */
+__device__ __forceinline__ uint32_t e8_ld_u32(const uint8_t *p){
+ return (uint32_t)p[0] | ((uint32_t)p[1]<<8) | ((uint32_t)p[2]<<16) | ((uint32_t)p[3]<<24);
+}
+/* Mirrors e8_fp16_to_f32 rather than calling __half2float, so the two decoders
+ * cannot disagree on subnormals. */
+__device__ __forceinline__ float e8_fp16(const uint8_t *p){
+ uint16_t h = (uint16_t)p[0] | ((uint16_t)p[1]<<8);
+ uint32_t sign=(uint32_t)(h&0x8000)<<16, exp=(h>>10)&0x1F, man=h&0x3FF, bits;
+ if (!exp) bits = man ? (sign|((127u-15u+1u-1u)<<23)|(man<<13)) : sign;
+ else if (exp==31) bits = sign|0x7F800000u|(man<<13);
+ else bits = sign|((exp+112u)<<23)|(man<<13);
+ float f; memcpy(&f,&bits,4); return f;
+}
+/* Expand one 32-weight sub-block; mirrors e8_expand_sub in quant.h. */
+__device__ __forceinline__ void e8_expand_sub_dev(const uint8_t *blk, int ib, float d, float *out){
+ uint32_t word = e8_ld_u32(blk + COLI_E8_QK/4 + ib*4);
+ float db = d * (0.5f + (float)((word>>28)&0xF)) * 0.5f;
+ const uint8_t *idx = blk + ib*8;
+ for (int l=0;l<4;l++){
+ uint32_t seven=(word>>(7*l))&0x7F;
+ const uint8_t *g0=c_e8_grid[idx[l*2+0]], *g1=c_e8_grid[idx[l*2+1]];
+ int par=0;
+ for (int j=0;j<8;j++){
+ int neg = j<7 ? (int)((seven>>j)&1) : 0;
+ if (j<7) par^=neg; else neg=par; /* odd parity closes the block */
+ float mag = (j<4 ? (float)g0[j] : (float)g1[j-4]) * 0.5f;
+ out[l*8+j] = neg ? -mag*db : mag*db;
+ }
+ }
+}
+
__device__ static float weight_at(const void *weights, int fmt, size_t row, int i) {
const uint8_t *base = static_cast<const uint8_t *>(weights) + row;
if (fmt == 0) return reinterpret_cast<const float *>(base)[i];
@@ -133,7 +179,20 @@ __global__ static void quant_matmul(float *y, const float *x, const void *weight
float sum = 0.0f;
size_t row = (size_t)o * rb;
const float *xs = x + (size_t)s * I;
- if (fmt == 4) {
+ if (fmt == 6) {
+ /* E8/IQ3: decode is per 32-weight sub-block, so threads stride over
+ * sub-blocks rather than elements -- expanding once per 32 weights
+ * instead of redoing the word/parity work for every element. */
+ const uint8_t *wrow = static_cast<const uint8_t *>(weights) + row;
+ int nsub = (I + COLI_E8_SUB - 1) / COLI_E8_SUB;
+ for (int sb = threadIdx.x; sb < nsub; sb += blockDim.x) {
+ const uint8_t *blk = wrow + (size_t)(sb / (COLI_E8_QK/COLI_E8_SUB)) * COLI_E8_BBYTES;
+ float w[COLI_E8_SUB];
+ e8_expand_sub_dev(blk, sb % (COLI_E8_QK/COLI_E8_SUB), e8_fp16(blk+96), w);
+ int off = sb*COLI_E8_SUB, n = I-off < COLI_E8_SUB ? I-off : COLI_E8_SUB;
+ for (int k=0;k<n;k++) sum += xs[off+k]*w[k];
+ }
+ } else if (fmt == 4) {
/* Grouped int4: one f32 scale per gs elements along I (ng groups per row).
* Scale layout: scales[o*ng + g]. Each thread strides through I, applying
* the appropriate group scale as it crosses group boundaries. This matches
@@ -157,7 +216,60 @@ __global__ static void quant_matmul(float *y, const float *x, const void *weight
__syncthreads();
}
if (!threadIdx.x)
- y[(size_t)s * O + o] = (fmt && fmt != 4) ? partial[0] * scales[o] : partial[0];
+ y[(size_t)s * O + o] = (fmt && fmt != 4 && fmt != 6) ? partial[0] * scales[o] : partial[0];
+}
+
+/* fmt=6 activation rotation, y = Q^T x for Q = D*H/sqrt(n) (#452). One block per
+ * row; the power-of-two block is staged in shared memory, capping n at 4096
+ * floats -- which covers every block GLM produces (6144 -> 2048+4096, 1536 ->
+ * 512+1024). The sign stream is regenerated in-kernel from the same xorshift64*
+ * that quant.h's e8_signs uses, so no rotation data is stored or uploaded.
+ *
+ * Placement note: all routed experts of a layer share one gate/up input, so that
+ * rotation belongs to the CALLER (once per layer). This kernel exists for the
+ * down projection, whose input is the per-expert silu(gate)*up product and so
+ * cannot be shared -- mirroring colibri.c's split at moe(). */
+__global__ static void e8_rot_rows_kernel(float *rows, int dim, int off, int n){
+ extern __shared__ float sh[];
+ __shared__ uint8_t sbits[4096/8];
+ if (!threadIdx.x) {
+ uint64_t s = 417u + (uint64_t)n;
+ for (int i=0;i<(n+7)/8;i++){
+ s^=s>>12; s^=s<<25; s^=s>>27;
+ sbits[i] = (uint8_t)((s*2685821657736338717ULL)>>56);
+ }
+ }
+ __syncthreads();
+ float *row = rows + (size_t)blockIdx.x*dim + off;
+ for (int i=threadIdx.x;i<n;i+=blockDim.x){
+ float v=row[i];
+ sh[i] = (sbits[i>>3]>>(i&7)&1) ? -v : v;
+ }
+ __syncthreads();
+ for (int len=1;len<n;len<<=1){
+ for (int j=threadIdx.x;j<n/2;j+=blockDim.x){
+ int i = (j/len)*(len<<1) + (j%len);
+ float u=sh[i], v=sh[i+len];
+ sh[i]=u+v; sh[i+len]=u-v;
+ }
+ __syncthreads();
+ }
+ float sc=rsqrtf((float)n);
+ for (int i=threadIdx.x;i<n;i+=blockDim.x) row[i]=sh[i]*sc;
+}
+
+/* Rotate nr rows in place, tiling non-power-of-two dims block-diagonally exactly
+ * as e8_rot_rows does. Returns 0 if a block exceeds the shared-memory cap. */
+static int e8_rot_rows_dev(float *rows, int nr, int dim, cudaStream_t stream){
+ int off = 0;
+ while (off < dim) {
+ int rem = dim-off, b = rem & (-rem);
+ while (b > 4096) b >>= 1;
+ e8_rot_rows_kernel<<<(unsigned)nr, 256, (size_t)b*sizeof(float), stream>>>(rows, dim, off, b);
+ if (cudaGetLastError() != cudaSuccess) return 0;
+ off += b;
+ }
+ return 1;
}
__global__ static void silu_mul(float *gate, const float *up, size_t n) {
@@ -495,6 +607,20 @@ static int reserve_pinned(float **ptr,size_t *cap,size_t bytes){
if(!cuda_ok(cudaMallocHost(ptr,bytes),"pinned staging allocation"))return 0;*cap=bytes;return 1;
}
+/* Publish quant.h's E8 codebook to every configured device. __constant__ memory
+ * is per-device, so this walks the contexts; the engine calls it once after init
+ * rather than the backend carrying a second copy of the table that could drift
+ * from the CPU decoder's (#452). Safe to call before any fmt=6 upload only. */
+extern "C" int coli_cuda_e8_set_grid(const void *grid) {
+ if (!grid || g_nctx < 1) return 0;
+ for (int i = 0; i < g_nctx; i++) {
+ if (!select_ctx(&g_ctx[i])) return 0;
+ if (!cuda_ok(cudaMemcpyToSymbol(c_e8_grid, grid, sizeof(c_e8_grid)), "E8 codebook upload"))
+ return 0;
+ }
+ return 1;
+}
+
extern "C" int coli_cuda_init(const int *devices, int count) {
#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__)
/* #509: the ROCm runtime (comgr, MIOpen, roctracer) reads $TEMP as a temp-dir
@@ -625,7 +751,9 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
DeviceContext *ctx = find_ctx(device);
if (!weights || I < 1 || O < 1 || !select_ctx(ctx)) return 0;
size_t rb = row_bytes(fmt, I);
- if (!rb || (fmt && !scales)) return 0;
+ /* fmt=6 keeps its scales inside each 98-byte block, so it is the one
+ * quantized format that legitimately arrives with scales == NULL. */
+ if (!rb || (fmt && fmt != 6 && !scales)) return 0;
ColiCudaTensor *t = static_cast<ColiCudaTensor *>(std::calloc(1, sizeof(*t)));
if (!t) return 0;
t->fmt = fmt; t->I = I; t->O = O; t->device = device; t->weight_bytes = rb * (size_t)O;
@@ -640,16 +768,17 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
if(fmt==2||fmt==4){ /* same nibble layout: offset-binary -> signed in place */
offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes);
if(!cuda_ok(cudaGetLastError(),"int4 weight conversion")){coli_cuda_tensor_free(t);return 0;}}
- if (fmt) {
+ if (fmt && fmt != 6) {
if (!cuda_ok(cudaMalloc(&t->scales, t->scale_count * sizeof(float)), "scale allocation") ||
!cuda_ok(cudaMemcpy(t->scales, scales, t->scale_count * sizeof(float), cudaMemcpyHostToDevice), "scale upload")) {
coli_cuda_tensor_free(t);
return 0;
}
}
+ if (fmt == 6) t->scale_count = 0; /* in-block scales: nothing separate to track */
t->tracked = 1;
ctx->tensor_count++;
- ctx->tensor_bytes += t->weight_bytes + (fmt ? t->scale_count * sizeof(float) : 0);
+ ctx->tensor_bytes += t->weight_bytes + ((fmt && fmt != 6) ? t->scale_count * sizeof(float) : 0);
*tensor = t;
return 1;
}
@@ -665,7 +794,7 @@ extern "C" int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,
extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor,
const void *weights,
const float *scales) {
- if (!tensor || !weights || (tensor->fmt && !scales)) return 0;
+ if (!tensor || !weights || (tensor->fmt && tensor->fmt != 6 && !scales)) return 0;
DeviceContext *ctx=find_ctx(tensor->device);
if (!select_ctx(ctx)) return 0;
if (!cuda_ok(cudaMemcpy(tensor->weights,weights,tensor->weight_bytes,
@@ -676,7 +805,9 @@ extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor,
if(!cuda_ok(cudaGetLastError(),"int4 weight refresh")) return 0;
}
int ng = tensor->ng > 0 ? tensor->ng : 1;
- return !tensor->fmt || cuda_ok(cudaMemcpy(tensor->scales,scales,
+ /* fmt=6 has no scale buffer at all (scales live in-block, scale_count 0), and
+ * the fallback below would otherwise copy O floats out of a NULL host pointer. */
+ return !tensor->fmt || tensor->fmt==6 || cuda_ok(cudaMemcpy(tensor->scales,scales,
(tensor->scale_count?tensor->scale_count:(size_t)tensor->O)*sizeof(float),
cudaMemcpyHostToDevice),"scale refresh");
}
@@ -746,6 +877,10 @@ extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,
up->fmt,S,D,I,row_bytes(up->fmt,D),up->gs,up->ng);
size_t n=(size_t)S*I;
silu_mul<<<(unsigned)((n+255)/256),256>>>(ctx->gate,ctx->up,n);
+ /* fmt=6: the down projection stores W@Q, so its input needs Q^T applied. This
+ * one is per-expert (the silu product is not shared), unlike the gate/up input
+ * rotation, which the caller does once per layer -- same split as moe(). */
+ if (down->fmt == 6 && !e8_rot_rows_dev(ctx->gate, S, I, 0)) return 0;
quant_matmul<<<output_grid,256>>>(ctx->y,ctx->gate,down->weights,down->scales,
down->fmt,S,I,D,row_bytes(down->fmt,I),down->gs,down->ng);
if (!cuda_ok(cudaGetLastError(),"expert MLP launch") ||
@@ -1159,7 +1294,11 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {
if (ctx) select_ctx(ctx);
if (tensor->tracked && ctx) {
int ng = tensor->ng > 0 ? tensor->ng : 1;
- size_t bytes = tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * ng * sizeof(float) : 0);
+ /* Must mirror the upload's accounting exactly: fmt=6 never charged for a
+ * scale buffer, and over-subtracting here trips the >= guard below, which
+ * silently leaves the tensor's bytes on the device counter forever. */
+ size_t bytes = tensor->weight_bytes +
+ ((tensor->fmt && tensor->fmt != 6) ? (size_t)tensor->O * ng * sizeof(float) : 0);
if (ctx->tensor_count) ctx->tensor_count--;
if (ctx->tensor_bytes >= bytes) ctx->tensor_bytes -= bytes;
}
diff --git a/c/backend_cuda.h b/c/backend_cuda.h
index b83c58a9..e3198368 100644
--- a/c/backend_cuda.h
+++ b/c/backend_cuda.h
@@ -35,6 +35,11 @@ COLI_CUDA_DLLEXPORT void coli_cuda_stats(int device, size_t *tensor_count, size_
COLI_CUDA_DLLEXPORT void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64_t *rows,
double *h2d_ms, double *kernel_ms, double *d2h_ms);
+/* Publish the E8 codebook (quant.h's e8_grid, 256x4 bytes) to every configured
+ * device. Must be called after coli_cuda_init and before any fmt=6 upload; the
+ * backend keeps no copy of the table so it cannot drift from the CPU decoder. */
+COLI_CUDA_DLLEXPORT int coli_cuda_e8_set_grid(const void *grid);
+
/* Upload without executing, so capacity failures happen during model startup. */
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,
const void *weights, const float *scales,
diff --git a/c/backend_gpu_compat.h b/c/backend_gpu_compat.h
index 91b9f285..74bbd578 100644
--- a/c/backend_gpu_compat.h
+++ b/c/backend_gpu_compat.h
@@ -47,6 +47,7 @@ namespace nvcuda { namespace wmma = ::rocwmma; }
#define cudaMemcpy hipMemcpy
#define cudaMemcpy2D hipMemcpy2D
#define cudaMemcpyAsync hipMemcpyAsync
+#define cudaMemcpyToSymbol hipMemcpyToSymbol /* fmt=6 E8 codebook upload */
#define cudaMemcpyHostToDevice hipMemcpyHostToDevice
#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost
#define cudaMemGetInfo hipMemGetInfo
diff --git a/c/backend_loader.c b/c/backend_loader.c
index 21be5484..abf0d99e 100644
--- a/c/backend_loader.c
+++ b/c/backend_loader.c
@@ -52,6 +52,7 @@ typedef int (*fn_attention_absorb)(ColiCudaTensor *kv_b, float *ctx,
typedef int (*fn_tensor_upload)(ColiCudaTensor **tensor, const void *weights,
const float *scales, int fmt, int I, int O, int device);
typedef int (*fn_tensor_upload_g)(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs);
+typedef int (*fn_e8_set_grid)(const void *grid);
typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const float *x,
const void *weights, const float *scales,
int fmt, int S, int I, int O, int device, int gs);
@@ -114,6 +115,7 @@ static struct {
fn_attention_absorb attention_absorb;
fn_tensor_upload tensor_upload;
fn_tensor_upload_g tensor_upload_g;
+ fn_e8_set_grid e8_set_grid;
fn_matmul matmul;
fn_tensor_free tensor_free;
fn_tensor_bytes tensor_bytes;
@@ -200,6 +202,15 @@ static int coli_cuda_load(void){
fprintf(stderr, "[CUDA] coli_cuda.dll missing symbol coli_cuda_" #name "\n"); \
FreeLibrary(g_cuda.dll); g_cuda.dll=NULL; return 0; }
+ /* Optional symbol: a DLL predating it leaves the pointer NULL and only that
+ * feature degrades (fmt=6 tensors stay CPU-side), instead of taking the whole
+ * GPU backend down over one missing entry point. */
+ #define RESOLVE_OPT(name, type) \
+ _Pragma("GCC diagnostic push") \
+ _Pragma("GCC diagnostic ignored \"-Wcast-function-type\"") \
+ g_cuda.name = (type)GetProcAddress(g_cuda.dll, "coli_cuda_" #name); \
+ _Pragma("GCC diagnostic pop")
+
RESOLVE(init, fn_init)
RESOLVE(shutdown, fn_shutdown)
RESOLVE(device_count, fn_device_count)
@@ -214,6 +225,7 @@ static int coli_cuda_load(void){
RESOLVE(attention_absorb, fn_attention_absorb)
RESOLVE(tensor_upload, fn_tensor_upload)
RESOLVE(tensor_upload_g, fn_tensor_upload_g)
+ RESOLVE_OPT(e8_set_grid, fn_e8_set_grid)
RESOLVE(matmul, fn_matmul)
RESOLVE(tensor_free, fn_tensor_free)
RESOLVE(tensor_bytes, fn_tensor_bytes)
@@ -341,6 +353,11 @@ int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, const void *weights, cons
return g_cuda.tensor_upload_g(tensor, weights, scales, fmt, I, O, device, gs);
}
+int coli_cuda_e8_set_grid(const void *grid){
+ if(!g_cuda.available || !g_cuda.e8_set_grid) return 0; /* fmt=6 stays CPU-side */
+ return g_cuda.e8_set_grid(grid);
+}
+
int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x,
const void *weights, const float *scales,
int fmt, int S, int I, int O, int device, int gs){
diff --git a/c/colibri.c b/c/colibri.c
index 53f6e799..8a978767 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -296,8 +296,10 @@ static void qt_cuda_reset(QT *t){
if(t->cuda){ coli_cuda_tensor_free(t->cuda); t->cuda=NULL; }
t->cuda_failed=0;
}
+static int g_cuda_e8_ready; /* codebook published to the devices (see cuda_boot) */
static int qt_cuda_upload(QT *t){
- if(t->fmt==5||t->fmt==6) return 0; /* int3-g64 / E8: no CUDA kernel yet β€” tensor stays CPU-side */
+ if(t->fmt==5) return 0; /* int3-g64: no CUDA kernel yet β€” tensor stays CPU-side */
+ if(t->fmt==6 && !g_cuda_e8_ready) return 0; /* E8 without its codebook would decode garbage */
const void *weights = t->fmt==0 ? (const void*)t->qf
: t->fmt==1 ? (const void*)t->q8 : (const void*)t->q4;
if(t->fmt==4) /* grouped int4 (#334): scales are [O, ceil(I/gs)] β€” the plain
@@ -3157,6 +3159,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
float *xg=falloc((int64_t)S*D), *gg=falloc((int64_t)S*I), *uu=falloc((int64_t)S*I), *hh=falloc((int64_t)S*D);
float *xe=NULL; /* fmt=6: x under the rotation Q^T, built once per call β€” all routed
* experts of the layer share it (the placement rule in quant.h) */
+ /* Materialise xe on first use. Every site that feeds a routed expert's gate/up
+ * input β€” CPU, the per-expert GPU call, and all three group packing paths β€”
+ * must go through here: doing the transform per expert instead of per layer
+ * costs ~11 ms against ~1.4 ms on GLM dims (#452). */
+ #define E8_XE(e) ((e)->g.fmt==6 ? (xe ? xe : (xe=falloc((int64_t)S*D), \
+ memcpy(xe,x,(size_t)S*D*sizeof(float)), e8_rot_rows(xe,S,D), xe)) : x)
int *rows=xalloc((size_t)S*sizeof(int),"moe rows"); float *rw=xalloc((size_t)S*sizeof(float),"moe rw");
#ifdef COLI_CUDA
/* PIPE Inc.1b: il batch-union del prefill passa dai gruppi GPU β€” prima di
@@ -3339,8 +3347,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
int nc=pd_nc[di]++; ESlot *e=pg_e[q];
pd_g[di][nc]=e->g.cuda; pd_u[di][nc]=e->u.cuda; pd_d[di][nc]=e->d.cuda;
pd_rows[di][nc]=pg_n[q]; pd_which[di][nc]=q;
+ const float *xsrc=E8_XE(e); /* fmt=6 feeds the rotated copy */
for(int r=0;r<pg_n[q];r++) memcpy(group_x+(int64_t)(pd_off[di]+cursor+r)*D,
- x+(int64_t)prow[q][r]*D,D*sizeof(float));
+ xsrc+(int64_t)prow[q][r]*D,D*sizeof(float));
cursor+=pg_n[q];
}
}
@@ -3469,11 +3478,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
ngroup++; continue;
}
#endif
- const float *xsrc=x;
- if(e->g.fmt==6){
- if(!xe){ xe=falloc((int64_t)S*D); memcpy(xe,x,(size_t)S*D*sizeof(float)); e8_rot_rows(xe,S,D); }
- xsrc=xe;
- }
+ const float *xsrc=E8_XE(e);
for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, xsrc+(int64_t)rows[r]*D, D*sizeof(float));
double t0=now_s();
#ifdef COLI_CUDA
@@ -3542,8 +3547,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
int nc=dev_nc[di]++; ESlot *e=group_e[q];
dev_g[di][nc]=e->g.cuda; dev_u[di][nc]=e->u.cuda; dev_d[di][nc]=e->d.cuda;
dev_rows[di][nc]=group_n[q]; dev_which[di][nc]=q;
+ const float *xsrc=E8_XE(e); /* fmt=6 feeds the rotated copy */
for(int r=0;r<group_n[q];r++) memcpy(group_x+(int64_t)(dev_off[di]+cursor+r)*D,
- x+(int64_t)group_row[(int64_t)q*S+r]*D,D*sizeof(float));
+ xsrc+(int64_t)group_row[(int64_t)q*S+r]*D,D*sizeof(float));
cursor+=group_n[q];
}
}
@@ -3611,12 +3617,14 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
for(int q=0;q<dev_nc[di];q++){
int gi=dev_which[di][q],nr=group_n[gi]; ESlot *e=group_e[gi];
if(!dev_ok[di]){
- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D,x+(int64_t)group_row[(int64_t)gi*S+r]*D,D*sizeof(float));
+ const float *xsrc=E8_XE(e); /* fmt=6 feeds the rotated copy */
+ for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D,xsrc+(int64_t)group_row[(int64_t)gi*S+r]*D,D*sizeof(float));
double tc=g_prof?now_s():0;
if(!coli_cuda_expert_mlp(e->g.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){
expert_host_ensure(m,layer,e);
expert_gate_up(gg,uu,xg,&e->g,&e->u,nr);
for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];
+ if(e->d.fmt==6) e8_rot_rows(gg,nr,I); /* down input, as on the main CPU path */
matmul_qt(hh,gg,&e->d,nr);
if(g_prof){m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d);
m->cpu_expert_rows+=(uint64_t)nr;}
@@ -3678,6 +3686,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
shared_done:
free(logits_all); free(choice); free(idxs); free(ws); free(keff); free(uniq);
free(xg); free(gg); free(uu); free(hh); free(rows); free(rw); free(xe);
+ #undef E8_XE
#ifdef COLI_CUDA
free(group_x);free(group_y);
free(group_row); free(group_weight);
@@ -6817,6 +6826,10 @@ int main(int argc, char **argv){
if(g_cuda_ndev<1){ fprintf(stderr,"invalid COLI_GPUS: use a list such as 0,1,2\n"); return 2; }
g_cuda_enabled=coli_cuda_init(g_cuda_devices,g_cuda_ndev);
if(!g_cuda_enabled){ fprintf(stderr,"[CUDA] requested backend is unavailable\n"); return 2; }
+ /* fmt=6 decodes against quant.h's codebook; publish it to every device so
+ * the backend never keeps a second copy that could drift (#452). An older
+ * DLL without the symbol leaves this 0 and fmt=6 tensors stay CPU-side. */
+ g_cuda_e8_ready=coli_cuda_e8_set_grid(e8_grid);
}
g_cuda_dense=getenv("CUDA_DENSE")?atoi(getenv("CUDA_DENSE")):0;
g_cuda_pipe=getenv("COLI_CUDA_PIPE")?atoi(getenv("COLI_CUDA_PIPE")):0;
diff --git a/c/tests/test_backend_cuda.cu b/c/tests/test_backend_cuda.cu
index 1a826b8f..0253cf60 100644
--- a/c/tests/test_backend_cuda.cu
+++ b/c/tests/test_backend_cuda.cu
@@ -4,6 +4,7 @@
#include <cstdio>
#include <cstdint>
#include <cstdlib>
+#include <cstring>
#ifdef _WIN32
/* MSVC has no POSIX setenv/unsetenv */
@@ -29,6 +30,191 @@ static int relative_rms(const float *got,const float *want,int n,float limit){
if(r>limit){std::fprintf(stderr,"relative RMS %.5f exceeds %.5f\n",r,limit);return 0;} return 1;
}
+/* ---- fmt=6 (E8/IQ3) --------------------------------------------------------
+ * The reference below is written from the format description, not shared with
+ * quant.h's decoder, so a common-mode mistake in one cannot hide in the other.
+ * A super-block is 98 bytes per 256 weights: 64 codebook indices, then 8 words
+ * of (4x7 sign bits + a 4-bit sub-scale in the top nibble), then an fp16 scale.
+ * Two indices feed each group of 8 weights; the 8th sign is the parity of the
+ * other 7. Scales live in the block, so fmt=6 tensors carry no scale array. */
+#define T6_QK 256
+#define T6_SUB 32
+#define T6_BB 98
+
+static void t6_decode_row(const uint8_t *row, int I, float *w,
+ const uint8_t grid[256][4]) {
+ int nb = (I + T6_QK - 1) / T6_QK;
+ for (int b = 0; b < nb; b++) {
+ const uint8_t *blk = row + (size_t)b*T6_BB;
+ uint16_t h = (uint16_t)blk[96] | ((uint16_t)blk[97] << 8);
+ /* fp16 -> float, written out rather than reusing any helper */
+ uint32_t sg=(uint32_t)(h&0x8000)<<16, ex=(h>>10)&0x1F, mn=h&0x3FF, bits;
+ if (!ex) bits = mn ? (sg|((127u-15u)<<23)|(mn<<13)) : sg;
+ else if (ex==31) bits = sg|0x7F800000u|(mn<<13);
+ else bits = sg|((ex+112u)<<23)|(mn<<13);
+ float d; std::memcpy(&d,&bits,4);
+ for (int ib = 0; ib < T6_QK/T6_SUB; ib++) {
+ int base = b*T6_QK + ib*T6_SUB;
+ if (base >= I) return;
+ const uint8_t *wp = blk + 64 + ib*4;
+ uint32_t word = (uint32_t)wp[0] | ((uint32_t)wp[1]<<8) |
+ ((uint32_t)wp[2]<<16) | ((uint32_t)wp[3]<<24);
+ float db = d * (0.5f + (float)(word >> 28)) * 0.5f;
+ for (int l = 0; l < 4; l++) {
+ uint32_t sev = (word >> (7*l)) & 0x7F;
+ const uint8_t *ga = grid[blk[ib*8 + l*2]];
+ const uint8_t *gb = grid[blk[ib*8 + l*2 + 1]];
+ int parity = 0;
+ for (int j = 0; j < 8; j++) {
+ int idx = base + l*8 + j;
+ if (idx >= I) break;
+ int neg;
+ if (j < 7) { neg = (sev >> j) & 1; parity ^= neg; }
+ else { neg = parity; }
+ float mag = (float)(j < 4 ? ga[j] : gb[j-4]) * 0.5f;
+ w[idx] = neg ? -mag*db : mag*db;
+ }
+ }
+ }
+ }
+}
+
+/* Sign stream: xorshift64* seeded 417+n, one bit per element (quant.h's
+ * e8_signs regenerates the identical stream, and the converter drew it too). */
+static void t6_signs(uint8_t *bits, int n) {
+ uint64_t s = 417u + (uint64_t)n;
+ for (int i = 0; i < (n+7)/8; i++) {
+ s ^= s >> 12; s ^= s << 25; s ^= s >> 27;
+ bits[i] = (uint8_t)((s * 2685821657736338717ULL) >> 56);
+ }
+}
+/* y = Q^T x, Q = D*H/sqrt(n); non-power-of-two dims tile block-diagonally. */
+static void t6_rot(float *row, int dim) {
+ int off = 0;
+ while (off < dim) {
+ int rem = dim-off, n = rem & (-rem);
+ while (n > 4096) n >>= 1;
+ uint8_t bits[4096/8]; t6_signs(bits, n);
+ float *a = row + off;
+ for (int i = 0; i < n; i++) if (bits[i>>3]>>(i&7)&1) a[i] = -a[i];
+ for (int len = 1; len < n; len <<= 1)
+ for (int i = 0; i < n; i += len<<1)
+ for (int j = i; j < i+len; j++) {
+ float u=a[j], v=a[j+len]; a[j]=u+v; a[j+len]=u-v;
+ }
+ float sc = 1.0f/std::sqrt((float)n);
+ for (int i = 0; i < n; i++) a[i] *= sc;
+ off += n;
+ }
+}
+
+static uint32_t t6_rng_state = 0x2545F491u;
+static uint32_t t6_rng(void){ t6_rng_state ^= t6_rng_state<<13; t6_rng_state ^= t6_rng_state>>17;
+ t6_rng_state ^= t6_rng_state<<5; return t6_rng_state; }
+
+/* Build a random-but-valid fmt=6 tensor: every index and sign pattern is legal,
+ * only the fp16 scale is constrained so the comparison stays meaningful. */
+static void t6_fill(uint8_t *q, int I, int O) {
+ int nb = (I + T6_QK - 1) / T6_QK;
+ for (int o = 0; o < O; o++)
+ for (int b = 0; b < nb; b++) {
+ uint8_t *blk = q + ((size_t)o*nb + b)*T6_BB;
+ for (int i = 0; i < 96; i++) blk[i] = (uint8_t)t6_rng();
+ uint16_t h = (uint16_t)((t6_rng() & 0x03FF) | (uint32_t)((10 + t6_rng()%6) << 10));
+ blk[96] = (uint8_t)(h & 0xFF); blk[97] = (uint8_t)(h >> 8);
+ }
+}
+
+static int test_fmt6(int dev) {
+ /* A synthetic codebook: the engine passes quant.h's real table, but any 256x4
+ * byte table is valid, and a synthetic one keeps this test self-contained
+ * while still exercising the upload path the real table travels. */
+ static uint8_t grid[256][4];
+ for (int i = 0; i < 256; i++)
+ for (int j = 0; j < 4; j++) grid[i][j] = (uint8_t)((i*4 + j) % 17);
+ if (!coli_cuda_e8_set_grid(grid)) { std::fprintf(stderr,"e8 grid upload failed\n"); return 0; }
+
+ const int I = 256, O = 128, S = 2;
+ int nb = (I + T6_QK - 1) / T6_QK;
+ uint8_t *q = (uint8_t*)std::malloc((size_t)O*nb*T6_BB);
+ t6_fill(q, I, O);
+ float *x = (float*)std::malloc((size_t)S*I*sizeof(float));
+ for (int i = 0; i < S*I; i++) x[i] = std::sin((float)(i+1)*0.031f);
+
+ /* --- matmul --- */
+ float *want = (float*)std::calloc((size_t)S*O, sizeof(float));
+ float *wrow = (float*)std::malloc((size_t)I*sizeof(float));
+ for (int o = 0; o < O; o++) {
+ t6_decode_row(q + (size_t)o*nb*T6_BB, I, wrow, grid);
+ for (int s = 0; s < S; s++) {
+ double acc = 0;
+ for (int i = 0; i < I; i++) acc += (double)x[s*I+i]*(double)wrow[i];
+ want[s*O+o] = (float)acc;
+ }
+ }
+ float *got = (float*)std::malloc((size_t)S*O*sizeof(float));
+ ColiCudaTensor *t6 = nullptr;
+ int ok = coli_cuda_matmul(&t6, got, x, q, nullptr, 6, S, I, O, dev, 0);
+ if (!ok) { std::fprintf(stderr,"fmt=6 matmul rejected\n"); return 0; }
+ if (!relative_rms(got, want, S*O, 1e-4f)) { std::fprintf(stderr,"fmt=6 matmul mismatch\n"); return 0; }
+
+ /* --- expert MLP: gate/up, silu, the device-side down rotation, down ---
+ * The caller owns the gate/up input rotation (once per layer), so x goes in
+ * as-is; the backend must rotate the silu product before the down matmul. */
+ uint8_t *qu = (uint8_t*)std::malloc((size_t)O*nb*T6_BB);
+ t6_fill(qu, I, O);
+ int nbd = (O + T6_QK - 1) / T6_QK;
+ uint8_t *qd = (uint8_t*)std::malloc((size_t)I*nbd*T6_BB);
+ t6_fill(qd, O, I);
+
+ float *g = (float*)std::malloc((size_t)S*O*sizeof(float));
+ float *u = (float*)std::malloc((size_t)S*O*sizeof(float));
+ float *wr2 = (float*)std::malloc((size_t)O*sizeof(float));
+ for (int o = 0; o < O; o++) {
+ t6_decode_row(q + (size_t)o*nb*T6_BB, I, wrow, grid);
+ float *wu = (float*)std::malloc((size_t)I*sizeof(float));
+ t6_decode_row(qu + (size_t)o*nb*T6_BB, I, wu, grid);
+ for (int s = 0; s < S; s++) {
+ double a = 0, b = 0;
+ for (int i = 0; i < I; i++) { a += (double)x[s*I+i]*(double)wrow[i];
+ b += (double)x[s*I+i]*(double)wu[i]; }
+ g[s*O+o] = (float)a; u[s*O+o] = (float)b;
+ }
+ std::free(wu);
+ }
+ for (int i = 0; i < S*O; i++) g[i] = (g[i]/(1.0f+std::exp(-g[i]))) * u[i];
+ for (int s = 0; s < S; s++) t6_rot(g + (size_t)s*O, O); /* down input */
+ float *want_e = (float*)std::calloc((size_t)S*I, sizeof(float));
+ for (int o = 0; o < I; o++) {
+ t6_decode_row(qd + (size_t)o*nbd*T6_BB, O, wr2, grid);
+ for (int s = 0; s < S; s++) {
+ double acc = 0;
+ for (int i = 0; i < O; i++) acc += (double)g[s*O+i]*(double)wr2[i];
+ want_e[s*I+o] = (float)acc;
+ }
+ }
+ ColiCudaTensor *tg6=nullptr,*tu6=nullptr,*td6=nullptr;
+ if (!coli_cuda_tensor_upload(&tg6,q, nullptr,6,I,O,dev) ||
+ !coli_cuda_tensor_upload(&tu6,qu,nullptr,6,I,O,dev) ||
+ !coli_cuda_tensor_upload(&td6,qd,nullptr,6,O,I,dev)) {
+ std::fprintf(stderr,"fmt=6 expert upload failed\n"); return 0;
+ }
+ float *got_e = (float*)std::malloc((size_t)S*I*sizeof(float));
+ if (!coli_cuda_expert_mlp(tg6,tu6,td6,got_e,x,S)) {
+ std::fprintf(stderr,"fmt=6 expert_mlp rejected\n"); return 0;
+ }
+ if (!relative_rms(got_e, want_e, S*I, 2e-4f)) {
+ std::fprintf(stderr,"fmt=6 expert_mlp mismatch (device-side rotation?)\n"); return 0;
+ }
+
+ coli_cuda_tensor_free(t6); coli_cuda_tensor_free(tg6);
+ coli_cuda_tensor_free(tu6); coli_cuda_tensor_free(td6);
+ std::free(q); std::free(qu); std::free(qd); std::free(x); std::free(want);
+ std::free(got); std::free(wrow); std::free(wr2); std::free(g); std::free(u);
+ std::free(want_e); std::free(got_e);
+ return 1;
+}
+
int main(int argc, char **argv) {
int devices[COLI_CUDA_MAX_DEVICES], ndev = argc > 1 ? argc - 1 : 1;
if (ndev > COLI_CUDA_MAX_DEVICES) return 2;
@@ -202,7 +388,13 @@ int main(int argc, char **argv) {
coli_cuda_tensor_free(td);
coli_cuda_stats(-1, &count, &bytes);
if (count || bytes) return 1;
+
+ /* fmt=6 runs after the stats assertions above, which pin exact tensor counts. */
+ if (!test_fmt6(d0)) return 1;
+ coli_cuda_stats(-1, &count, &bytes);
+ if (count || bytes) { std::fprintf(stderr,"fmt=6 leaked tensors\n"); return 1; }
+
coli_cuda_shutdown();
- std::printf("cuda backend: q8/q4/q2/f32 correctness ok on %d device(s)\n", ndev);
+ std::printf("cuda backend: q8/q4/q2/f32/e8 correctness ok on %d device(s)\n", ndev);
return 0;
}