Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
cuda: fmt=6 (E8/IQ3) decode, so E8 containers can reach VRAM
ZacharyZcRce25594bde7b
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex 71355c25..5a739cd9 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -87,15 +87,61 @@ static int select_ctx(DeviceContext *ctx) {return 1;}+/* fmt=6 (E8/IQ3) geometry, mirroring quant.h. A super-block packs 256 weights+ * into 98 bytes: 64 codebook indices, 8 words of (4x7 signs + 4-bit sub-scale),+ * and one fp16 super-scale. Scales live INSIDE the block, so fmt=6 tensors carry+ * no separate scale array (#452). */+#define COLI_E8_QK 256+#define COLI_E8_SUB 32+#define COLI_E8_BBYTES 98+__host__ __device__ static size_t row_bytes(int fmt, int I) {if (fmt == 0) return (size_t)I * sizeof(float);if (fmt == 1) return (size_t)I;if (fmt == 2 || fmt == 4) return (size_t)(I + 1) / 2; /* fmt=4: same packed int4 */if (fmt == 3) return (size_t)(I + 3) / 4;if (fmt == 4) return (size_t)(I + 1) / 2; /* grouped int4: nibbles like fmt 2 */+ if (fmt == 6) return (size_t)(((int64_t)I + COLI_E8_QK - 1) / COLI_E8_QK) * COLI_E8_BBYTES;return 0;}+/* The E8 codebook, uploaded once per device from quant.h's e8_grid so the table+ * has a single source of truth and cannot drift from the CPU decoder. */+__constant__ uint8_t c_e8_grid[256][4];++/* A super-block is 98 bytes, so nothing inside it is guaranteed 4- or 2-byte+ * aligned: assemble the words byte-wise instead of dereferencing. */+__device__ __forceinline__ uint32_t e8_ld_u32(const uint8_t *p){+ return (uint32_t)p[0] | ((uint32_t)p[1]<<8) | ((uint32_t)p[2]<<16) | ((uint32_t)p[3]<<24);+}+/* Mirrors e8_fp16_to_f32 rather than calling __half2float, so the two decoders+ * cannot disagree on subnormals. */+__device__ __forceinline__ float e8_fp16(const uint8_t *p){+ uint16_t h = (uint16_t)p[0] | ((uint16_t)p[1]<<8);+ uint32_t sign=(uint32_t)(h&0x8000)<<16, exp=(h>>10)&0x1F, man=h&0x3FF, bits;+ if (!exp) bits = man ? (sign|((127u-15u+1u-1u)<<23)|(man<<13)) : sign;+ else if (exp==31) bits = sign|0x7F800000u|(man<<13);+ else bits = sign|((exp+112u)<<23)|(man<<13);+ float f; memcpy(&f,&bits,4); return f;+}+/* Expand one 32-weight sub-block; mirrors e8_expand_sub in quant.h. */+__device__ __forceinline__ void e8_expand_sub_dev(const uint8_t *blk, int ib, float d, float *out){+ uint32_t word = e8_ld_u32(blk + COLI_E8_QK/4 + ib*4);+ float db = d * (0.5f + (float)((word>>28)&0xF)) * 0.5f;+ const uint8_t *idx = blk + ib*8;+ for (int l=0;l<4;l++){+ uint32_t seven=(word>>(7*l))&0x7F;+ const uint8_t *g0=c_e8_grid[idx[l*2+0]], *g1=c_e8_grid[idx[l*2+1]];+ int par=0;+ for (int j=0;j<8;j++){+ int neg = j<7 ? (int)((seven>>j)&1) : 0;+ if (j<7) par^=neg; else neg=par; /* odd parity closes the block */+ float mag = (j<4 ? (float)g0[j] : (float)g1[j-4]) * 0.5f;+ out[l*8+j] = neg ? -mag*db : mag*db;+ }+ }+}+__device__ static float weight_at(const void *weights, int fmt, size_t row, int i) {const uint8_t *base = static_cast<const uint8_t *>(weights) + row;if (fmt == 0) return reinterpret_cast<const float *>(base)[i];@@ -133,7 +179,20 @@ __global__ static void quant_matmul(float *y, const float *x, const void *weightfloat sum = 0.0f;size_t row = (size_t)o * rb;const float *xs = x + (size_t)s * I;- if (fmt == 4) {+ if (fmt == 6) {+ /* E8/IQ3: decode is per 32-weight sub-block, so threads stride over+ * sub-blocks rather than elements -- expanding once per 32 weights+ * instead of redoing the word/parity work for every element. */+ const uint8_t *wrow = static_cast<const uint8_t *>(weights) + row;+ int nsub = (I + COLI_E8_SUB - 1) / COLI_E8_SUB;+ for (int sb = threadIdx.x; sb < nsub; sb += blockDim.x) {+ const uint8_t *blk = wrow + (size_t)(sb / (COLI_E8_QK/COLI_E8_SUB)) * COLI_E8_BBYTES;+ float w[COLI_E8_SUB];+ e8_expand_sub_dev(blk, sb % (COLI_E8_QK/COLI_E8_SUB), e8_fp16(blk+96), w);+ int off = sb*COLI_E8_SUB, n = I-off < COLI_E8_SUB ? I-off : COLI_E8_SUB;+ for (int k=0;k<n;k++) sum += xs[off+k]*w[k];+ }+ } else if (fmt == 4) {/* Grouped int4: one f32 scale per gs elements along I (ng groups per row).* Scale layout: scales[o*ng + g]. Each thread strides through I, applying* the appropriate group scale as it crosses group boundaries. This matches@@ -157,7 +216,60 @@ __global__ static void quant_matmul(float *y, const float *x, const void *weight__syncthreads();}if (!threadIdx.x)- y[(size_t)s * O + o] = (fmt && fmt != 4) ? partial[0] * scales[o] : partial[0];+ y[(size_t)s * O + o] = (fmt && fmt != 4 && fmt != 6) ? partial[0] * scales[o] : partial[0];+}++/* fmt=6 activation rotation, y = Q^T x for Q = D*H/sqrt(n) (#452). One block per+ * row; the power-of-two block is staged in shared memory, capping n at 4096+ * floats -- which covers every block GLM produces (6144 -> 2048+4096, 1536 ->+ * 512+1024). The sign stream is regenerated in-kernel from the same xorshift64*+ * that quant.h's e8_signs uses, so no rotation data is stored or uploaded.+ *+ * Placement note: all routed experts of a layer share one gate/up input, so that+ * rotation belongs to the CALLER (once per layer). This kernel exists for the+ * down projection, whose input is the per-expert silu(gate)*up product and so+ * cannot be shared -- mirroring colibri.c's split at moe(). */+__global__ static void e8_rot_rows_kernel(float *rows, int dim, int off, int n){+ extern __shared__ float sh[];+ __shared__ uint8_t sbits[4096/8];+ if (!threadIdx.x) {+ uint64_t s = 417u + (uint64_t)n;+ for (int i=0;i<(n+7)/8;i++){+ s^=s>>12; s^=s<<25; s^=s>>27;+ sbits[i] = (uint8_t)((s*2685821657736338717ULL)>>56);+ }+ }+ __syncthreads();+ float *row = rows + (size_t)blockIdx.x*dim + off;+ for (int i=threadIdx.x;i<n;i+=blockDim.x){+ float v=row[i];+ sh[i] = (sbits[i>>3]>>(i&7)&1) ? -v : v;+ }+ __syncthreads();+ for (int len=1;len<n;len<<=1){+ for (int j=threadIdx.x;j<n/2;j+=blockDim.x){+ int i = (j/len)*(len<<1) + (j%len);+ float u=sh[i], v=sh[i+len];+ sh[i]=u+v; sh[i+len]=u-v;+ }+ __syncthreads();+ }+ float sc=rsqrtf((float)n);+ for (int i=threadIdx.x;i<n;i+=blockDim.x) row[i]=sh[i]*sc;+}++/* Rotate nr rows in place, tiling non-power-of-two dims block-diagonally exactly+ * as e8_rot_rows does. Returns 0 if a block exceeds the shared-memory cap. */+static int e8_rot_rows_dev(float *rows, int nr, int dim, cudaStream_t stream){+ int off = 0;+ while (off < dim) {+ int rem = dim-off, b = rem & (-rem);+ while (b > 4096) b >>= 1;+ e8_rot_rows_kernel<<<(unsigned)nr, 256, (size_t)b*sizeof(float), stream>>>(rows, dim, off, b);+ if (cudaGetLastError() != cudaSuccess) return 0;+ off += b;+ }+ return 1;}__global__ static void silu_mul(float *gate, const float *up, size_t n) {@@ -495,6 +607,20 @@ static int reserve_pinned(float **ptr,size_t *cap,size_t bytes){if(!cuda_ok(cudaMallocHost(ptr,bytes),"pinned staging allocation"))return 0;*cap=bytes;return 1;}+/* Publish quant.h's E8 codebook to every configured device. __constant__ memory+ * is per-device, so this walks the contexts; the engine calls it once after init+ * rather than the backend carrying a second copy of the table that could drift+ * from the CPU decoder's (#452). Safe to call before any fmt=6 upload only. */+extern "C" int coli_cuda_e8_set_grid(const void *grid) {+ if (!grid || g_nctx < 1) return 0;+ for (int i = 0; i < g_nctx; i++) {+ if (!select_ctx(&g_ctx[i])) return 0;+ if (!cuda_ok(cudaMemcpyToSymbol(c_e8_grid, grid, sizeof(c_e8_grid)), "E8 codebook upload"))+ return 0;+ }+ return 1;+}+extern "C" int coli_cuda_init(const int *devices, int count) {#if defined(__HIP_PLATFORM_AMD__) || defined(__HIP__)/* #509: the ROCm runtime (comgr, MIOpen, roctracer) reads $TEMP as a temp-dir@@ -625,7 +751,9 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,DeviceContext *ctx = find_ctx(device);if (!weights || I < 1 || O < 1 || !select_ctx(ctx)) return 0;size_t rb = row_bytes(fmt, I);- if (!rb || (fmt && !scales)) return 0;+ /* fmt=6 keeps its scales inside each 98-byte block, so it is the one+ * quantized format that legitimately arrives with scales == NULL. */+ if (!rb || (fmt && fmt != 6 && !scales)) return 0;ColiCudaTensor *t = static_cast<ColiCudaTensor *>(std::calloc(1, sizeof(*t)));if (!t) return 0;t->fmt = fmt; t->I = I; t->O = O; t->device = device; t->weight_bytes = rb * (size_t)O;@@ -640,16 +768,17 @@ extern "C" int coli_cuda_tensor_upload(ColiCudaTensor **tensor,if(fmt==2||fmt==4){ /* same nibble layout: offset-binary -> signed in place */offset_to_signed_s4<<<(unsigned)((t->weight_bytes+255)/256),256>>>((uint8_t*)t->weights,t->weight_bytes);if(!cuda_ok(cudaGetLastError(),"int4 weight conversion")){coli_cuda_tensor_free(t);return 0;}}- if (fmt) {+ if (fmt && fmt != 6) {if (!cuda_ok(cudaMalloc(&t->scales, t->scale_count * sizeof(float)), "scale allocation") ||!cuda_ok(cudaMemcpy(t->scales, scales, t->scale_count * sizeof(float), cudaMemcpyHostToDevice), "scale upload")) {coli_cuda_tensor_free(t);return 0;}}+ if (fmt == 6) t->scale_count = 0; /* in-block scales: nothing separate to track */t->tracked = 1;ctx->tensor_count++;- ctx->tensor_bytes += t->weight_bytes + (fmt ? t->scale_count * sizeof(float) : 0);+ ctx->tensor_bytes += t->weight_bytes + ((fmt && fmt != 6) ? t->scale_count * sizeof(float) : 0);*tensor = t;return 1;}@@ -665,7 +794,7 @@ extern "C" int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor,const void *weights,const float *scales) {- if (!tensor || !weights || (tensor->fmt && !scales)) return 0;+ if (!tensor || !weights || (tensor->fmt && tensor->fmt != 6 && !scales)) return 0;DeviceContext *ctx=find_ctx(tensor->device);if (!select_ctx(ctx)) return 0;if (!cuda_ok(cudaMemcpy(tensor->weights,weights,tensor->weight_bytes,@@ -676,7 +805,9 @@ extern "C" int coli_cuda_tensor_update(ColiCudaTensor *tensor,if(!cuda_ok(cudaGetLastError(),"int4 weight refresh")) return 0;}int ng = tensor->ng > 0 ? tensor->ng : 1;- return !tensor->fmt || cuda_ok(cudaMemcpy(tensor->scales,scales,+ /* fmt=6 has no scale buffer at all (scales live in-block, scale_count 0), and+ * the fallback below would otherwise copy O floats out of a NULL host pointer. */+ return !tensor->fmt || tensor->fmt==6 || cuda_ok(cudaMemcpy(tensor->scales,scales,(tensor->scale_count?tensor->scale_count:(size_t)tensor->O)*sizeof(float),cudaMemcpyHostToDevice),"scale refresh");}@@ -746,6 +877,10 @@ extern "C" int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,up->fmt,S,D,I,row_bytes(up->fmt,D),up->gs,up->ng);size_t n=(size_t)S*I;silu_mul<<<(unsigned)((n+255)/256),256>>>(ctx->gate,ctx->up,n);+ /* fmt=6: the down projection stores W@Q, so its input needs Q^T applied. This+ * one is per-expert (the silu product is not shared), unlike the gate/up input+ * rotation, which the caller does once per layer -- same split as moe(). */+ if (down->fmt == 6 && !e8_rot_rows_dev(ctx->gate, S, I, 0)) return 0;quant_matmul<<<output_grid,256>>>(ctx->y,ctx->gate,down->weights,down->scales,down->fmt,S,I,D,row_bytes(down->fmt,I),down->gs,down->ng);if (!cuda_ok(cudaGetLastError(),"expert MLP launch") ||@@ -1159,7 +1294,11 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {if (ctx) select_ctx(ctx);if (tensor->tracked && ctx) {int ng = tensor->ng > 0 ? tensor->ng : 1;- size_t bytes = tensor->weight_bytes + (tensor->fmt ? (size_t)tensor->O * ng * sizeof(float) : 0);+ /* Must mirror the upload's accounting exactly: fmt=6 never charged for a+ * scale buffer, and over-subtracting here trips the >= guard below, which+ * silently leaves the tensor's bytes on the device counter forever. */+ size_t bytes = tensor->weight_bytes ++ ((tensor->fmt && tensor->fmt != 6) ? (size_t)tensor->O * ng * sizeof(float) : 0);if (ctx->tensor_count) ctx->tensor_count--;if (ctx->tensor_bytes >= bytes) ctx->tensor_bytes -= bytes;}diff --git a/c/backend_cuda.h b/c/backend_cuda.hindex b83c58a9..e3198368 100644--- a/c/backend_cuda.h+++ b/c/backend_cuda.h@@ -35,6 +35,11 @@ COLI_CUDA_DLLEXPORT void coli_cuda_stats(int device, size_t *tensor_count, size_COLI_CUDA_DLLEXPORT void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64_t *rows,double *h2d_ms, double *kernel_ms, double *d2h_ms);+/* Publish the E8 codebook (quant.h's e8_grid, 256x4 bytes) to every configured+ * device. Must be called after coli_cuda_init and before any fmt=6 upload; the+ * backend keeps no copy of the table so it cannot drift from the CPU decoder. */+COLI_CUDA_DLLEXPORT int coli_cuda_e8_set_grid(const void *grid);+/* Upload without executing, so capacity failures happen during model startup. */COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,const void *weights, const float *scales,diff --git a/c/backend_gpu_compat.h b/c/backend_gpu_compat.hindex 91b9f285..74bbd578 100644--- a/c/backend_gpu_compat.h+++ b/c/backend_gpu_compat.h@@ -47,6 +47,7 @@ namespace nvcuda { namespace wmma = ::rocwmma; }#define cudaMemcpy hipMemcpy#define cudaMemcpy2D hipMemcpy2D#define cudaMemcpyAsync hipMemcpyAsync+#define cudaMemcpyToSymbol hipMemcpyToSymbol /* fmt=6 E8 codebook upload */#define cudaMemcpyHostToDevice hipMemcpyHostToDevice#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost#define cudaMemGetInfo hipMemGetInfodiff --git a/c/backend_loader.c b/c/backend_loader.cindex 21be5484..abf0d99e 100644--- a/c/backend_loader.c+++ b/c/backend_loader.c@@ -52,6 +52,7 @@ typedef int (*fn_attention_absorb)(ColiCudaTensor *kv_b, float *ctx,typedef int (*fn_tensor_upload)(ColiCudaTensor **tensor, const void *weights,const float *scales, int fmt, int I, int O, int device);typedef int (*fn_tensor_upload_g)(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs);+typedef int (*fn_e8_set_grid)(const void *grid);typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const float *x,const void *weights, const float *scales,int fmt, int S, int I, int O, int device, int gs);@@ -114,6 +115,7 @@ static struct {fn_attention_absorb attention_absorb;fn_tensor_upload tensor_upload;fn_tensor_upload_g tensor_upload_g;+ fn_e8_set_grid e8_set_grid;fn_matmul matmul;fn_tensor_free tensor_free;fn_tensor_bytes tensor_bytes;@@ -200,6 +202,15 @@ static int coli_cuda_load(void){fprintf(stderr, "[CUDA] coli_cuda.dll missing symbol coli_cuda_" #name "\n"); \FreeLibrary(g_cuda.dll); g_cuda.dll=NULL; return 0; }+ /* Optional symbol: a DLL predating it leaves the pointer NULL and only that+ * feature degrades (fmt=6 tensors stay CPU-side), instead of taking the whole+ * GPU backend down over one missing entry point. */+ #define RESOLVE_OPT(name, type) \+ _Pragma("GCC diagnostic push") \+ _Pragma("GCC diagnostic ignored \"-Wcast-function-type\"") \+ g_cuda.name = (type)GetProcAddress(g_cuda.dll, "coli_cuda_" #name); \+ _Pragma("GCC diagnostic pop")+RESOLVE(init, fn_init)RESOLVE(shutdown, fn_shutdown)RESOLVE(device_count, fn_device_count)@@ -214,6 +225,7 @@ static int coli_cuda_load(void){RESOLVE(attention_absorb, fn_attention_absorb)RESOLVE(tensor_upload, fn_tensor_upload)RESOLVE(tensor_upload_g, fn_tensor_upload_g)+ RESOLVE_OPT(e8_set_grid, fn_e8_set_grid)RESOLVE(matmul, fn_matmul)RESOLVE(tensor_free, fn_tensor_free)RESOLVE(tensor_bytes, fn_tensor_bytes)@@ -341,6 +353,11 @@ int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, const void *weights, consreturn g_cuda.tensor_upload_g(tensor, weights, scales, fmt, I, O, device, gs);}+int coli_cuda_e8_set_grid(const void *grid){+ if(!g_cuda.available || !g_cuda.e8_set_grid) return 0; /* fmt=6 stays CPU-side */+ return g_cuda.e8_set_grid(grid);+}+int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x,const void *weights, const float *scales,int fmt, int S, int I, int O, int device, int gs){diff --git a/c/colibri.c b/c/colibri.cindex 53f6e799..8a978767 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -296,8 +296,10 @@ static void qt_cuda_reset(QT *t){if(t->cuda){ coli_cuda_tensor_free(t->cuda); t->cuda=NULL; }t->cuda_failed=0;}+static int g_cuda_e8_ready; /* codebook published to the devices (see cuda_boot) */static int qt_cuda_upload(QT *t){- if(t->fmt==5||t->fmt==6) return 0; /* int3-g64 / E8: no CUDA kernel yet β tensor stays CPU-side */+ if(t->fmt==5) return 0; /* int3-g64: no CUDA kernel yet β tensor stays CPU-side */+ if(t->fmt==6 && !g_cuda_e8_ready) return 0; /* E8 without its codebook would decode garbage */const void *weights = t->fmt==0 ? (const void*)t->qf: t->fmt==1 ? (const void*)t->q8 : (const void*)t->q4;if(t->fmt==4) /* grouped int4 (#334): scales are [O, ceil(I/gs)] β the plain@@ -3157,6 +3159,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intfloat *xg=falloc((int64_t)S*D), *gg=falloc((int64_t)S*I), *uu=falloc((int64_t)S*I), *hh=falloc((int64_t)S*D);float *xe=NULL; /* fmt=6: x under the rotation Q^T, built once per call β all routed* experts of the layer share it (the placement rule in quant.h) */+ /* Materialise xe on first use. Every site that feeds a routed expert's gate/up+ * input β CPU, the per-expert GPU call, and all three group packing paths β+ * must go through here: doing the transform per expert instead of per layer+ * costs ~11 ms against ~1.4 ms on GLM dims (#452). */+ #define E8_XE(e) ((e)->g.fmt==6 ? (xe ? xe : (xe=falloc((int64_t)S*D), \+ memcpy(xe,x,(size_t)S*D*sizeof(float)), e8_rot_rows(xe,S,D), xe)) : x)int *rows=xalloc((size_t)S*sizeof(int),"moe rows"); float *rw=xalloc((size_t)S*sizeof(float),"moe rw");#ifdef COLI_CUDA/* PIPE Inc.1b: il batch-union del prefill passa dai gruppi GPU β prima di@@ -3339,8 +3347,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intint nc=pd_nc[di]++; ESlot *e=pg_e[q];pd_g[di][nc]=e->g.cuda; pd_u[di][nc]=e->u.cuda; pd_d[di][nc]=e->d.cuda;pd_rows[di][nc]=pg_n[q]; pd_which[di][nc]=q;+ const float *xsrc=E8_XE(e); /* fmt=6 feeds the rotated copy */for(int r=0;r<pg_n[q];r++) memcpy(group_x+(int64_t)(pd_off[di]+cursor+r)*D,- x+(int64_t)prow[q][r]*D,D*sizeof(float));+ xsrc+(int64_t)prow[q][r]*D,D*sizeof(float));cursor+=pg_n[q];}}@@ -3469,11 +3478,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intngroup++; continue;}#endif- const float *xsrc=x;- if(e->g.fmt==6){- if(!xe){ xe=falloc((int64_t)S*D); memcpy(xe,x,(size_t)S*D*sizeof(float)); e8_rot_rows(xe,S,D); }- xsrc=xe;- }+ const float *xsrc=E8_XE(e);for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, xsrc+(int64_t)rows[r]*D, D*sizeof(float));double t0=now_s();#ifdef COLI_CUDA@@ -3542,8 +3547,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intint nc=dev_nc[di]++; ESlot *e=group_e[q];dev_g[di][nc]=e->g.cuda; dev_u[di][nc]=e->u.cuda; dev_d[di][nc]=e->d.cuda;dev_rows[di][nc]=group_n[q]; dev_which[di][nc]=q;+ const float *xsrc=E8_XE(e); /* fmt=6 feeds the rotated copy */for(int r=0;r<group_n[q];r++) memcpy(group_x+(int64_t)(dev_off[di]+cursor+r)*D,- x+(int64_t)group_row[(int64_t)q*S+r]*D,D*sizeof(float));+ xsrc+(int64_t)group_row[(int64_t)q*S+r]*D,D*sizeof(float));cursor+=group_n[q];}}@@ -3611,12 +3617,14 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intfor(int q=0;q<dev_nc[di];q++){int gi=dev_which[di][q],nr=group_n[gi]; ESlot *e=group_e[gi];if(!dev_ok[di]){- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D,x+(int64_t)group_row[(int64_t)gi*S+r]*D,D*sizeof(float));+ const float *xsrc=E8_XE(e); /* fmt=6 feeds the rotated copy */+ for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D,xsrc+(int64_t)group_row[(int64_t)gi*S+r]*D,D*sizeof(float));double tc=g_prof?now_s():0;if(!coli_cuda_expert_mlp(e->g.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){expert_host_ensure(m,layer,e);expert_gate_up(gg,uu,xg,&e->g,&e->u,nr);for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];+ if(e->d.fmt==6) e8_rot_rows(gg,nr,I); /* down input, as on the main CPU path */matmul_qt(hh,gg,&e->d,nr);if(g_prof){m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d);m->cpu_expert_rows+=(uint64_t)nr;}@@ -3678,6 +3686,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intshared_done:free(logits_all); free(choice); free(idxs); free(ws); free(keff); free(uniq);free(xg); free(gg); free(uu); free(hh); free(rows); free(rw); free(xe);+ #undef E8_XE#ifdef COLI_CUDAfree(group_x);free(group_y);free(group_row); free(group_weight);@@ -6817,6 +6826,10 @@ int main(int argc, char **argv){if(g_cuda_ndev<1){ fprintf(stderr,"invalid COLI_GPUS: use a list such as 0,1,2\n"); return 2; }g_cuda_enabled=coli_cuda_init(g_cuda_devices,g_cuda_ndev);if(!g_cuda_enabled){ fprintf(stderr,"[CUDA] requested backend is unavailable\n"); return 2; }+ /* fmt=6 decodes against quant.h's codebook; publish it to every device so+ * the backend never keeps a second copy that could drift (#452). An older+ * DLL without the symbol leaves this 0 and fmt=6 tensors stay CPU-side. */+ g_cuda_e8_ready=coli_cuda_e8_set_grid(e8_grid);}g_cuda_dense=getenv("CUDA_DENSE")?atoi(getenv("CUDA_DENSE")):0;g_cuda_pipe=getenv("COLI_CUDA_PIPE")?atoi(getenv("COLI_CUDA_PIPE")):0;diff --git a/c/tests/test_backend_cuda.cu b/c/tests/test_backend_cuda.cuindex 1a826b8f..0253cf60 100644--- a/c/tests/test_backend_cuda.cu+++ b/c/tests/test_backend_cuda.cu@@ -4,6 +4,7 @@#include <cstdio>#include <cstdint>#include <cstdlib>+#include <cstring>#ifdef _WIN32/* MSVC has no POSIX setenv/unsetenv */@@ -29,6 +30,191 @@ static int relative_rms(const float *got,const float *want,int n,float limit){if(r>limit){std::fprintf(stderr,"relative RMS %.5f exceeds %.5f\n",r,limit);return 0;} return 1;}+/* ---- fmt=6 (E8/IQ3) --------------------------------------------------------+ * The reference below is written from the format description, not shared with+ * quant.h's decoder, so a common-mode mistake in one cannot hide in the other.+ * A super-block is 98 bytes per 256 weights: 64 codebook indices, then 8 words+ * of (4x7 sign bits + a 4-bit sub-scale in the top nibble), then an fp16 scale.+ * Two indices feed each group of 8 weights; the 8th sign is the parity of the+ * other 7. Scales live in the block, so fmt=6 tensors carry no scale array. */+#define T6_QK 256+#define T6_SUB 32+#define T6_BB 98++static void t6_decode_row(const uint8_t *row, int I, float *w,+ const uint8_t grid[256][4]) {+ int nb = (I + T6_QK - 1) / T6_QK;+ for (int b = 0; b < nb; b++) {+ const uint8_t *blk = row + (size_t)b*T6_BB;+ uint16_t h = (uint16_t)blk[96] | ((uint16_t)blk[97] << 8);+ /* fp16 -> float, written out rather than reusing any helper */+ uint32_t sg=(uint32_t)(h&0x8000)<<16, ex=(h>>10)&0x1F, mn=h&0x3FF, bits;+ if (!ex) bits = mn ? (sg|((127u-15u)<<23)|(mn<<13)) : sg;+ else if (ex==31) bits = sg|0x7F800000u|(mn<<13);+ else bits = sg|((ex+112u)<<23)|(mn<<13);+ float d; std::memcpy(&d,&bits,4);+ for (int ib = 0; ib < T6_QK/T6_SUB; ib++) {+ int base = b*T6_QK + ib*T6_SUB;+ if (base >= I) return;+ const uint8_t *wp = blk + 64 + ib*4;+ uint32_t word = (uint32_t)wp[0] | ((uint32_t)wp[1]<<8) |+ ((uint32_t)wp[2]<<16) | ((uint32_t)wp[3]<<24);+ float db = d * (0.5f + (float)(word >> 28)) * 0.5f;+ for (int l = 0; l < 4; l++) {+ uint32_t sev = (word >> (7*l)) & 0x7F;+ const uint8_t *ga = grid[blk[ib*8 + l*2]];+ const uint8_t *gb = grid[blk[ib*8 + l*2 + 1]];+ int parity = 0;+ for (int j = 0; j < 8; j++) {+ int idx = base + l*8 + j;+ if (idx >= I) break;+ int neg;+ if (j < 7) { neg = (sev >> j) & 1; parity ^= neg; }+ else { neg = parity; }+ float mag = (float)(j < 4 ? ga[j] : gb[j-4]) * 0.5f;+ w[idx] = neg ? -mag*db : mag*db;+ }+ }+ }+ }+}++/* Sign stream: xorshift64* seeded 417+n, one bit per element (quant.h's+ * e8_signs regenerates the identical stream, and the converter drew it too). */+static void t6_signs(uint8_t *bits, int n) {+ uint64_t s = 417u + (uint64_t)n;+ for (int i = 0; i < (n+7)/8; i++) {+ s ^= s >> 12; s ^= s << 25; s ^= s >> 27;+ bits[i] = (uint8_t)((s * 2685821657736338717ULL) >> 56);+ }+}+/* y = Q^T x, Q = D*H/sqrt(n); non-power-of-two dims tile block-diagonally. */+static void t6_rot(float *row, int dim) {+ int off = 0;+ while (off < dim) {+ int rem = dim-off, n = rem & (-rem);+ while (n > 4096) n >>= 1;+ uint8_t bits[4096/8]; t6_signs(bits, n);+ float *a = row + off;+ for (int i = 0; i < n; i++) if (bits[i>>3]>>(i&7)&1) a[i] = -a[i];+ for (int len = 1; len < n; len <<= 1)+ for (int i = 0; i < n; i += len<<1)+ for (int j = i; j < i+len; j++) {+ float u=a[j], v=a[j+len]; a[j]=u+v; a[j+len]=u-v;+ }+ float sc = 1.0f/std::sqrt((float)n);+ for (int i = 0; i < n; i++) a[i] *= sc;+ off += n;+ }+}++static uint32_t t6_rng_state = 0x2545F491u;+static uint32_t t6_rng(void){ t6_rng_state ^= t6_rng_state<<13; t6_rng_state ^= t6_rng_state>>17;+ t6_rng_state ^= t6_rng_state<<5; return t6_rng_state; }++/* Build a random-but-valid fmt=6 tensor: every index and sign pattern is legal,+ * only the fp16 scale is constrained so the comparison stays meaningful. */+static void t6_fill(uint8_t *q, int I, int O) {+ int nb = (I + T6_QK - 1) / T6_QK;+ for (int o = 0; o < O; o++)+ for (int b = 0; b < nb; b++) {+ uint8_t *blk = q + ((size_t)o*nb + b)*T6_BB;+ for (int i = 0; i < 96; i++) blk[i] = (uint8_t)t6_rng();+ uint16_t h = (uint16_t)((t6_rng() & 0x03FF) | (uint32_t)((10 + t6_rng()%6) << 10));+ blk[96] = (uint8_t)(h & 0xFF); blk[97] = (uint8_t)(h >> 8);+ }+}++static int test_fmt6(int dev) {+ /* A synthetic codebook: the engine passes quant.h's real table, but any 256x4+ * byte table is valid, and a synthetic one keeps this test self-contained+ * while still exercising the upload path the real table travels. */+ static uint8_t grid[256][4];+ for (int i = 0; i < 256; i++)+ for (int j = 0; j < 4; j++) grid[i][j] = (uint8_t)((i*4 + j) % 17);+ if (!coli_cuda_e8_set_grid(grid)) { std::fprintf(stderr,"e8 grid upload failed\n"); return 0; }++ const int I = 256, O = 128, S = 2;+ int nb = (I + T6_QK - 1) / T6_QK;+ uint8_t *q = (uint8_t*)std::malloc((size_t)O*nb*T6_BB);+ t6_fill(q, I, O);+ float *x = (float*)std::malloc((size_t)S*I*sizeof(float));+ for (int i = 0; i < S*I; i++) x[i] = std::sin((float)(i+1)*0.031f);++ /* --- matmul --- */+ float *want = (float*)std::calloc((size_t)S*O, sizeof(float));+ float *wrow = (float*)std::malloc((size_t)I*sizeof(float));+ for (int o = 0; o < O; o++) {+ t6_decode_row(q + (size_t)o*nb*T6_BB, I, wrow, grid);+ for (int s = 0; s < S; s++) {+ double acc = 0;+ for (int i = 0; i < I; i++) acc += (double)x[s*I+i]*(double)wrow[i];+ want[s*O+o] = (float)acc;+ }+ }+ float *got = (float*)std::malloc((size_t)S*O*sizeof(float));+ ColiCudaTensor *t6 = nullptr;+ int ok = coli_cuda_matmul(&t6, got, x, q, nullptr, 6, S, I, O, dev, 0);+ if (!ok) { std::fprintf(stderr,"fmt=6 matmul rejected\n"); return 0; }+ if (!relative_rms(got, want, S*O, 1e-4f)) { std::fprintf(stderr,"fmt=6 matmul mismatch\n"); return 0; }++ /* --- expert MLP: gate/up, silu, the device-side down rotation, down ---+ * The caller owns the gate/up input rotation (once per layer), so x goes in+ * as-is; the backend must rotate the silu product before the down matmul. */+ uint8_t *qu = (uint8_t*)std::malloc((size_t)O*nb*T6_BB);+ t6_fill(qu, I, O);+ int nbd = (O + T6_QK - 1) / T6_QK;+ uint8_t *qd = (uint8_t*)std::malloc((size_t)I*nbd*T6_BB);+ t6_fill(qd, O, I);++ float *g = (float*)std::malloc((size_t)S*O*sizeof(float));+ float *u = (float*)std::malloc((size_t)S*O*sizeof(float));+ float *wr2 = (float*)std::malloc((size_t)O*sizeof(float));+ for (int o = 0; o < O; o++) {+ t6_decode_row(q + (size_t)o*nb*T6_BB, I, wrow, grid);+ float *wu = (float*)std::malloc((size_t)I*sizeof(float));+ t6_decode_row(qu + (size_t)o*nb*T6_BB, I, wu, grid);+ for (int s = 0; s < S; s++) {+ double a = 0, b = 0;+ for (int i = 0; i < I; i++) { a += (double)x[s*I+i]*(double)wrow[i];+ b += (double)x[s*I+i]*(double)wu[i]; }+ g[s*O+o] = (float)a; u[s*O+o] = (float)b;+ }+ std::free(wu);+ }+ for (int i = 0; i < S*O; i++) g[i] = (g[i]/(1.0f+std::exp(-g[i]))) * u[i];+ for (int s = 0; s < S; s++) t6_rot(g + (size_t)s*O, O); /* down input */+ float *want_e = (float*)std::calloc((size_t)S*I, sizeof(float));+ for (int o = 0; o < I; o++) {+ t6_decode_row(qd + (size_t)o*nbd*T6_BB, O, wr2, grid);+ for (int s = 0; s < S; s++) {+ double acc = 0;+ for (int i = 0; i < O; i++) acc += (double)g[s*O+i]*(double)wr2[i];+ want_e[s*I+o] = (float)acc;+ }+ }+ ColiCudaTensor *tg6=nullptr,*tu6=nullptr,*td6=nullptr;+ if (!coli_cuda_tensor_upload(&tg6,q, nullptr,6,I,O,dev) ||+ !coli_cuda_tensor_upload(&tu6,qu,nullptr,6,I,O,dev) ||+ !coli_cuda_tensor_upload(&td6,qd,nullptr,6,O,I,dev)) {+ std::fprintf(stderr,"fmt=6 expert upload failed\n"); return 0;+ }+ float *got_e = (float*)std::malloc((size_t)S*I*sizeof(float));+ if (!coli_cuda_expert_mlp(tg6,tu6,td6,got_e,x,S)) {+ std::fprintf(stderr,"fmt=6 expert_mlp rejected\n"); return 0;+ }+ if (!relative_rms(got_e, want_e, S*I, 2e-4f)) {+ std::fprintf(stderr,"fmt=6 expert_mlp mismatch (device-side rotation?)\n"); return 0;+ }++ coli_cuda_tensor_free(t6); coli_cuda_tensor_free(tg6);+ coli_cuda_tensor_free(tu6); coli_cuda_tensor_free(td6);+ std::free(q); std::free(qu); std::free(qd); std::free(x); std::free(want);+ std::free(got); std::free(wrow); std::free(wr2); std::free(g); std::free(u);+ std::free(want_e); std::free(got_e);+ return 1;+}+int main(int argc, char **argv) {int devices[COLI_CUDA_MAX_DEVICES], ndev = argc > 1 ? argc - 1 : 1;if (ndev > COLI_CUDA_MAX_DEVICES) return 2;@@ -202,7 +388,13 @@ int main(int argc, char **argv) {coli_cuda_tensor_free(td);coli_cuda_stats(-1, &count, &bytes);if (count || bytes) return 1;++ /* fmt=6 runs after the stats assertions above, which pin exact tensor counts. */+ if (!test_fmt6(d0)) return 1;+ coli_cuda_stats(-1, &count, &bytes);+ if (count || bytes) { std::fprintf(stderr,"fmt=6 leaked tensors\n"); return 1; }+coli_cuda_shutdown();- std::printf("cuda backend: q8/q4/q2/f32 correctness ok on %d device(s)\n", ndev);+ std::printf("cuda backend: q8/q4/q2/f32/e8 correctness ok on %d device(s)\n", ndev);return 0;}