Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
backend: the gated-delta step takes a gate mode and a split in_proj
kreuzzelg4bbc228784ce
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cuindex d1bd3efd..a61e3933 100644--- a/c/backend_cuda.cu+++ b/c/backend_cuda.cu@@ -1806,7 +1806,7 @@ extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor,struct DnGates { float egh[DN_MAX_HEADS]; float beta[DN_MAX_HEADS]; };struct ColiCudaDn {int device, vh, vk, kdim, vdim, conv_dim, convk, hidden;- float eps;+ float eps; int gate_sigmoid;float *ring, *rec, *conv_w, *norm_w; /* resident */float *x, *qkvz, *conv_out, *outr, *out; /* scratch for one token */};@@ -1850,7 +1850,7 @@ __device__ static double dn_block_sum(double v, double *sh) {__global__ void __launch_bounds__(256) dn_head_kernel(const float *__restrict__ conv_out, const float *__restrict__ z,float *__restrict__ S, float *__restrict__ outr,const float *__restrict__ normw, DnGates g,- int rep, int kdim, int vdim, int key_dim_tot, float scale, float eps) {+ int rep, int kdim, int vdim, int key_dim_tot, float scale, float eps, int gate_sigmoid) {__shared__ float q[256], k[256], delta[256];__shared__ double red[32];int h = blockIdx.x, t = threadIdx.x;@@ -1881,13 +1881,14 @@ __global__ void __launch_bounds__(256) dn_head_kernel(const float *__restrict__if (t < vdim) {float r = 1.f / sqrtf((float)(ms / vdim) + eps);float zz = z[(size_t)h * vdim + t];- outr[(size_t)h * vdim + t] = (o * r * normw[t]) * zz / (1.f + expf(-zz));+ float gate = gate_sigmoid ? 1.f / (1.f + expf(-zz)) : zz / (1.f + expf(-zz));+ outr[(size_t)h * vdim + t] = (o * r * normw[t]) * gate;}}extern "C" ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim,int conv_dim, int convk, int hidden,- const float *conv_w, const float *norm_w, float eps) {+ const float *conv_w, const float *norm_w, float eps, int gate_sigmoid) {if (vh < 1 || vh > DN_MAX_HEADS || vk < 1 || vh % vk || kdim < 1 || kdim > 256 || vdim < 1 || vdim > 256 ||convk < 2 || conv_dim != 2 * vk * kdim + vh * vdim || hidden < 1 || !conv_w || !norm_w) return NULL;DeviceContext *ctx = find_ctx(device);@@ -1895,7 +1896,7 @@ extern "C" ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim,ColiCudaDn *d = static_cast<ColiCudaDn *>(std::calloc(1, sizeof(*d)));if (!d) return NULL;d->device = device; d->vh = vh; d->vk = vk; d->kdim = kdim; d->vdim = vdim;- d->conv_dim = conv_dim; d->convk = convk; d->hidden = hidden; d->eps = eps;+ d->conv_dim = conv_dim; d->convk = convk; d->hidden = hidden; d->eps = eps; d->gate_sigmoid = gate_sigmoid ? 1 : 0;size_t ring_b = (size_t)conv_dim * (convk - 1) * sizeof(float), rec_b = (size_t)vh * kdim * vdim * sizeof(float);size_t proj = (size_t)conv_dim + (size_t)vh * vdim;int ok = cuda_ok(cudaMalloc(&d->ring, ring_b), "dn ring") && cuda_ok(cudaMalloc(&d->rec, rec_b), "dn state") &&@@ -1937,25 +1938,30 @@ extern "C" int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec) {return cuda_ok(cudaMemcpy(ring, d->ring, ring_b, cudaMemcpyDeviceToHost), "dn ring get") &&cuda_ok(cudaMemcpy(rec, d->rec, rec_b, cudaMemcpyDeviceToHost), "dn state get");}-extern "C" int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp,+extern "C" int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *projz, ColiCudaTensor *outp,const float *x, float *out, const float *egh, const float *beta) {if (fault_injected()) return 0;if (!d || !proj || !outp || !x || !out || !egh || !beta) return 0;size_t proj_dim = (size_t)d->conv_dim + (size_t)d->vh * d->vdim, value_dim = (size_t)d->vh * d->vdim;- if (proj->fmt != 1 || proj->I != d->hidden || (size_t)proj->O != proj_dim || proj->device != d->device ||+ size_t want_o = projz ? (size_t)d->conv_dim : proj_dim;+ if (proj->fmt != 1 || proj->I != d->hidden || (size_t)proj->O != want_o || proj->device != d->device ||outp->fmt != 1 || (size_t)outp->I != value_dim || outp->O != d->hidden || outp->device != d->device) return 0;+ if (projz && (projz->fmt != 1 || projz->I != d->hidden || (size_t)projz->O != value_dim || projz->device != d->device)) return 0;if (!select_ctx(find_ctx(d->device))) return 0;DnGates g;for (int h = 0; h < d->vh; h++) { g.egh[h] = egh[h]; g.beta[h] = beta[h]; }if (!cuda_ok(cudaMemcpy(d->x, x, (size_t)d->hidden * sizeof(float), cudaMemcpyHostToDevice), "dn x upload")) return 0;- quant_matmul_launch(d->qkvz, d->x, proj->weights, proj->scales, 1, 1, d->hidden, (int)proj_dim,+ quant_matmul_launch(d->qkvz, d->x, proj->weights, proj->scales, 1, 1, d->hidden, (int)want_o,row_bytes(1, d->hidden), proj->gs, proj->ng);+ if (projz)+ quant_matmul_launch(d->qkvz + d->conv_dim, d->x, projz->weights, projz->scales, 1, 1, d->hidden, (int)value_dim,+ row_bytes(1, d->hidden), projz->gs, projz->ng);dn_conv_kernel<<<(unsigned)((d->conv_dim + 255) / 256), 256>>>(d->qkvz, d->ring, d->conv_w, d->conv_out, d->conv_dim, d->convk);int threads = d->kdim > d->vdim ? d->kdim : d->vdim;threads = ((threads + 31) / 32) * 32;dn_head_kernel<<<(unsigned)d->vh, threads>>>(d->conv_out, d->qkvz + d->conv_dim, d->rec, d->outr, d->norm_w, g,d->vh / d->vk, d->kdim, d->vdim, d->vk * d->kdim,- 1.f / sqrtf((float)d->kdim), d->eps);+ 1.f / sqrtf((float)d->kdim), d->eps, d->gate_sigmoid);quant_matmul_launch(d->out, d->outr, outp->weights, outp->scales, 1, 1, (int)value_dim, d->hidden,row_bytes(1, (int)value_dim), outp->gs, outp->ng);return cuda_ok(cudaGetLastError(), "dn step launch") &&diff --git a/c/backend_cuda.h b/c/backend_cuda.hindex 9ecf6976..7ee39fe7 100644--- a/c/backend_cuda.h+++ b/c/backend_cuda.h@@ -155,11 +155,15 @@ COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload(ColiCudaTensor **tensor,typedef struct ColiCudaDn ColiCudaDn;COLI_CUDA_DLLEXPORT ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim,int conv_dim, int convk, int hidden,- const float *conv_w, const float *norm_w, float eps);+ const float *conv_w, const float *norm_w, float eps, int gate_sigmoid);COLI_CUDA_DLLEXPORT void coli_cuda_dn_free(ColiCudaDn *d);COLI_CUDA_DLLEXPORT int coli_cuda_dn_set_state(ColiCudaDn *d, const float *ring, const float *rec); /* NULL = zero */COLI_CUDA_DLLEXPORT int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec);-COLI_CUDA_DLLEXPORT int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp,+/* proj: the in_proj rows. Fused qkv ++ z in one matrix (projz NULL, qwen36's+ * dnproj) or qkv alone with z in projz (qwen38's dnqkv / dnz). gate_sigmoid at+ * create: the gated RMSNorm multiplies by silu(z) (0, Qwen3.6) or sigmoid(z)+ * (1, Qwen3.8). */+COLI_CUDA_DLLEXPORT int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *projz, ColiCudaTensor *outp,const float *x, float *out, const float *egh, const float *beta);#ifdef COLI_ANS/* Experimental Linux-only GPU-resident entropy tier. The archive remains indiff --git a/c/backend_loader.c b/c/backend_loader.cindex f990135d..4cb1e4e9 100644--- a/c/backend_loader.c+++ b/c/backend_loader.c@@ -95,11 +95,11 @@ typedef int (*fn_attention_absorb)(ColiCudaTensor *kv_b, float *ctx,typedef int (*fn_tensor_upload)(ColiCudaTensor **tensor, const void *weights,const float *scales, int fmt, int I, int O, int device);typedef int (*fn_tensor_upload_g)(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs);-typedef ColiCudaDn *(*fn_dn_create)(int device, int vh, int vk, int kdim, int vdim, int conv_dim, int convk, int hidden, const float *conv_w, const float *norm_w, float eps);+typedef ColiCudaDn *(*fn_dn_create)(int device, int vh, int vk, int kdim, int vdim, int conv_dim, int convk, int hidden, const float *conv_w, const float *norm_w, float eps, int gate_sigmoid);typedef void (*fn_dn_free)(ColiCudaDn *d);typedef int (*fn_dn_set_state)(ColiCudaDn *d, const float *ring, const float *rec);typedef int (*fn_dn_get_state)(ColiCudaDn *d, float *ring, float *rec);-typedef int (*fn_dn_step)(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta);+typedef int (*fn_dn_step)(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *projz, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta);typedef int (*fn_e8_set_grid)(const void *grid);typedef int (*fn_fp8_set_lut)(const float *lut);typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const float *x,@@ -1653,15 +1653,15 @@ int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, const void *weights, cons/* Gated delta layer on the device: every entry point is optional, a DLL* predating them answers NULL/0 and the engine keeps the layer on the CPU. */-ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim, int conv_dim, int convk, int hidden, const float *conv_w, const float *norm_w, float eps){+ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim, int conv_dim, int convk, int hidden, const float *conv_w, const float *norm_w, float eps, int gate_sigmoid){if(!g_cuda.available || !g_cuda.dn_create || !g_cuda.dn_free || !g_cuda.dn_set_state || !g_cuda.dn_get_state || !g_cuda.dn_step) return NULL;- return g_cuda.dn_create(device, vh, vk, kdim, vdim, conv_dim, convk, hidden, conv_w, norm_w, eps);+ return g_cuda.dn_create(device, vh, vk, kdim, vdim, conv_dim, convk, hidden, conv_w, norm_w, eps, gate_sigmoid);}void coli_cuda_dn_free(ColiCudaDn *d){ if(d && g_cuda.dn_free) g_cuda.dn_free(d); }int coli_cuda_dn_set_state(ColiCudaDn *d, const float *ring, const float *rec){ return d && g_cuda.dn_set_state ? g_cuda.dn_set_state(d, ring, rec) : 0; }int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec){ return d && g_cuda.dn_get_state ? g_cuda.dn_get_state(d, ring, rec) : 0; }-int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta){- return d && g_cuda.dn_step ? g_cuda.dn_step(d, proj, outp, x, out, egh, beta) : 0;+int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *projz, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta){+ return d && g_cuda.dn_step ? g_cuda.dn_step(d, proj, projz, outp, x, out, egh, beta) : 0;}int coli_cuda_e8_set_grid(const void *grid){diff --git a/c/tests/qwen36_fake_cuda.h b/c/tests/qwen36_fake_cuda.hindex c1c5b9dd..a7097b5c 100644--- a/c/tests/qwen36_fake_cuda.h+++ b/c/tests/qwen36_fake_cuda.h@@ -136,7 +136,7 @@ int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x, const vo* kernels (and as qwen36.c's deltanet()), so an engine test can put the layer* on the fake tier and demand the same tokens as the CPU path. Matmuls go* through the fake's fmt=1 compute when fake_dense_compute is on. */-struct ColiCudaDn { int device, vh, vk, kdim, vdim, conv_dim, convk, hidden; float eps;+struct ColiCudaDn { int device, vh, vk, kdim, vdim, conv_dim, convk, hidden; float eps; int gate_sigmoid;float *ring, *rec, *conv_w, *norm_w; };static int fake_dn_steps, fake_dn_fail;static void fake_gemv_i8(float *y, const float *x, const ColiCudaTensor *t) {@@ -145,10 +145,10 @@ static void fake_gemv_i8(float *y, const float *x, const ColiCudaTensor *t) {for (int i = 0; i < t->I; i++) a += x[i] * (float)w[i]; y[o] = a * t->sc[o]; }}ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim, int conv_dim, int convk, int hidden,- const float *conv_w, const float *norm_w, float eps) {+ const float *conv_w, const float *norm_w, float eps, int gate_sigmoid) {if (fake_dn_fail || vh < 1 || vk < 1 || vh % vk || convk < 2 || conv_dim != 2 * vk * kdim + vh * vdim || !conv_w || !norm_w) return NULL;ColiCudaDn *d = (ColiCudaDn *)calloc(1, sizeof *d);- d->device = device; d->vh = vh; d->vk = vk; d->kdim = kdim; d->vdim = vdim; d->conv_dim = conv_dim; d->convk = convk; d->hidden = hidden; d->eps = eps;+ d->device = device; d->vh = vh; d->vk = vk; d->kdim = kdim; d->vdim = vdim; d->conv_dim = conv_dim; d->convk = convk; d->hidden = hidden; d->eps = eps; d->gate_sigmoid = gate_sigmoid;d->ring = (float *)calloc((size_t)conv_dim * (convk - 1), sizeof(float));d->rec = (float *)calloc((size_t)vh * kdim * vdim, sizeof(float));d->conv_w = (float *)malloc((size_t)conv_dim * convk * sizeof(float)); memcpy(d->conv_w, conv_w, (size_t)conv_dim * convk * sizeof(float));@@ -169,15 +169,17 @@ int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec) {memcpy(rec, d->rec, (size_t)d->vh * d->kdim * d->vdim * sizeof(float));return 1;}-int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta) {+int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *projz, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta) {if (!d || !proj || !outp || !x || !out || fake_dn_fail) return 0;- if (!fake_dense_compute || !proj->w || !outp->w) return 0; /* nothing to compute with: the engine keeps the CPU path */+ if (!fake_dense_compute || !proj->w || !outp->w || (projz && !projz->w)) return 0; /* nothing to compute with: the engine keeps the CPU path */int vh = d->vh, kdim = d->kdim, vdim = d->vdim, rep = vh / d->vk, key_dim_tot = d->vk * kdim, conv_dim = d->conv_dim, convk = d->convk;- size_t proj_dim = (size_t)conv_dim + (size_t)vh * vdim;- if (proj->fmt != 1 || proj->I != d->hidden || (size_t)proj->O != proj_dim || outp->fmt != 1 || (size_t)outp->I != (size_t)vh * vdim || outp->O != d->hidden) return 0;+ size_t proj_dim = (size_t)conv_dim + (size_t)vh * vdim, want_o = projz ? (size_t)conv_dim : proj_dim;+ if (proj->fmt != 1 || proj->I != d->hidden || (size_t)proj->O != want_o || outp->fmt != 1 || (size_t)outp->I != (size_t)vh * vdim || outp->O != d->hidden) return 0;+ if (projz && (projz->fmt != 1 || projz->I != d->hidden || (size_t)projz->O != (size_t)vh * vdim)) return 0;float *qkvz = (float *)malloc(proj_dim * sizeof(float)), *conv_out = (float *)malloc((size_t)conv_dim * sizeof(float));float *outr = (float *)malloc((size_t)vh * vdim * sizeof(float));fake_gemv_i8(qkvz, x, proj);+ if (projz) fake_gemv_i8(qkvz + conv_dim, x, projz);const float *z = qkvz + conv_dim;for (int cc = 0; cc < conv_dim; cc++) {const float *w = d->conv_w + (size_t)cc * convk; float *rg = d->ring + (size_t)cc * (convk - 1);@@ -199,7 +201,7 @@ int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp,for (int t = 0; t < vdim; t++) { float acc = 0.f; for (int kk = 0; kk < kdim; kk++) { float s = Sh[(size_t)kk * vdim + t] * egh[h] + k[kk] * delta[t]; Sh[(size_t)kk * vdim + t] = s; acc += q[kk] * s; } o[t] = acc; }double ms = 0; for (int t = 0; t < vdim; t++) ms += (double)o[t] * o[t];float r = 1.f / sqrtf((float)(ms / vdim) + d->eps);- for (int t = 0; t < vdim; t++) { float zz = z[(size_t)h * vdim + t]; outr[(size_t)h * vdim + t] = (o[t] * r * d->norm_w[t]) * zz / (1.f + expf(-zz)); }+ for (int t = 0; t < vdim; t++) { float zz = z[(size_t)h * vdim + t]; float gate = d->gate_sigmoid ? 1.f / (1.f + expf(-zz)) : zz / (1.f + expf(-zz)); outr[(size_t)h * vdim + t] = (o[t] * r * d->norm_w[t]) * gate; }}fake_gemv_i8(out, outr, outp);free(qkvz); free(conv_out); free(outr);