Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

backend: the gated delta layer on the device (coli_cuda_dn_*)

kreuzzelg34fdc4adefe5
diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu
index b7930a0f..7222bd75 100644
--- a/c/backend_cuda.cu
+++ b/c/backend_cuda.cu
@@ -1801,6 +1801,167 @@ extern "C" int coli_cuda_matmul(ColiCudaTensor **tensor,
return 1;
}
+/* ---- gated delta layer on the device (see backend_cuda.h) ---------------- */
+#define DN_MAX_HEADS 64
+struct DnGates { float egh[DN_MAX_HEADS]; float beta[DN_MAX_HEADS]; };
+struct ColiCudaDn {
+ int device, vh, vk, kdim, vdim, conv_dim, convk, hidden;
+ float eps;
+ float *ring, *rec, *conv_w, *norm_w; /* resident */
+ float *x, *qkvz, *conv_out, *outr, *out; /* scratch for one token */
+};
+
+/* one thread per channel: causal depthwise conv over the ring + this token,
+ * SiLU, then the ring advances (drop oldest, append this token's value) */
+__global__ void dn_conv_kernel(const float *__restrict__ qkv, float *__restrict__ ring,
+ const float *__restrict__ w, float *__restrict__ conv_out,
+ int conv_dim, int convk) {
+ int cc = blockIdx.x * blockDim.x + threadIdx.x;
+ if (cc >= conv_dim) return;
+ const float *wc = w + (size_t)cc * convk;
+ float *rg = ring + (size_t)cc * (convk - 1);
+ float acc = 0.f;
+ for (int kk = 0; kk < convk - 1; kk++) acc += wc[kk] * rg[kk];
+ float cur = qkv[cc];
+ acc += wc[convk - 1] * cur;
+ conv_out[cc] = acc / (1.f + expf(-acc));
+ for (int kk = 0; kk < convk - 2; kk++) rg[kk] = rg[kk + 1];
+ rg[convk - 2] = cur;
+}
+
+__device__ static double dn_block_sum(double v, double *sh) {
+ /* blockDim <= 1024; warp shuffle then one pass over warps */
+ for (int o = 16; o > 0; o >>= 1) v += __shfl_down_sync(0xffffffffu, v, o);
+ int lane = threadIdx.x & 31, wid = threadIdx.x >> 5;
+ __syncthreads();
+ if (lane == 0) sh[wid] = v;
+ __syncthreads();
+ double r = 0.0;
+ int nw = (blockDim.x + 31) >> 5;
+ for (int i = 0; i < nw; i++) r += sh[i];
+ return r;
+}
+
+/* one block per value head; thread t owns column t of the state (vdim) and
+ * element t of q/k (kdim). Same arithmetic as the CPU deltanet(): norms in
+ * double with eps inside the sqrt, S = S*egh + k delta^T with delta read from
+ * the decayed state, out = q^T S over the updated state, then the gated
+ * RMSNorm with silu(z). */
+__global__ void __launch_bounds__(256) dn_head_kernel(const float *__restrict__ conv_out, const float *__restrict__ z,
+ float *__restrict__ S, float *__restrict__ outr,
+ const float *__restrict__ normw, DnGates g,
+ int rep, int kdim, int vdim, int key_dim_tot, float scale, float eps) {
+ __shared__ float q[256], k[256], delta[256];
+ __shared__ double red[32];
+ int h = blockIdx.x, t = threadIdx.x;
+ int kh = h / rep;
+ const float *qs = conv_out + (size_t)kh * kdim;
+ const float *ks = conv_out + key_dim_tot + (size_t)kh * kdim;
+ const float *vs = conv_out + 2 * (size_t)key_dim_tot + (size_t)h * vdim;
+ float qv = t < kdim ? qs[t] : 0.f, kv_ = t < kdim ? ks[t] : 0.f;
+ double sq = dn_block_sum((double)qv * qv, red);
+ double sk = dn_block_sum((double)kv_ * kv_, red);
+ double nq = sqrt(sq + 1e-6), nk = sqrt(sk + 1e-6);
+ if (t < kdim) { q[t] = (float)((double)qv / nq * scale); k[t] = (float)((double)kv_ / nk); }
+ __syncthreads();
+ float egh = g.egh[h], beta = g.beta[h];
+ float *Sh = S + (size_t)h * kdim * vdim;
+ float o = 0.f;
+ if (t < vdim) {
+ float kvsum = 0.f;
+ for (int kk = 0; kk < kdim; kk++) kvsum += k[kk] * (Sh[(size_t)kk * vdim + t] * egh);
+ delta[t] = (vs[t] - kvsum) * beta;
+ for (int kk = 0; kk < kdim; kk++) {
+ float s = Sh[(size_t)kk * vdim + t] * egh + k[kk] * delta[t];
+ Sh[(size_t)kk * vdim + t] = s;
+ o += q[kk] * s;
+ }
+ }
+ double ms = dn_block_sum((double)o * o, red);
+ if (t < vdim) {
+ float r = 1.f / sqrtf((float)(ms / vdim) + eps);
+ float zz = z[(size_t)h * vdim + t];
+ outr[(size_t)h * vdim + t] = (o * r * normw[t]) * zz / (1.f + expf(-zz));
+ }
+}
+
+extern "C" ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim,
+ int conv_dim, int convk, int hidden,
+ const float *conv_w, const float *norm_w, float eps) {
+ if (vh < 1 || vh > DN_MAX_HEADS || vk < 1 || vh % vk || kdim < 1 || kdim > 256 || vdim < 1 || vdim > 256 ||
+ convk < 2 || conv_dim != 2 * vk * kdim + vh * vdim || hidden < 1 || !conv_w || !norm_w) return NULL;
+ DeviceContext *ctx = find_ctx(device);
+ if (!select_ctx(ctx)) return NULL;
+ ColiCudaDn *d = static_cast<ColiCudaDn *>(std::calloc(1, sizeof(*d)));
+ if (!d) return NULL;
+ d->device = device; d->vh = vh; d->vk = vk; d->kdim = kdim; d->vdim = vdim;
+ d->conv_dim = conv_dim; d->convk = convk; d->hidden = hidden; d->eps = eps;
+ size_t ring_b = (size_t)conv_dim * (convk - 1) * sizeof(float), rec_b = (size_t)vh * kdim * vdim * sizeof(float);
+ size_t proj = (size_t)conv_dim + (size_t)vh * vdim;
+ int ok = cuda_ok(cudaMalloc(&d->ring, ring_b), "dn ring") && cuda_ok(cudaMalloc(&d->rec, rec_b), "dn state") &&
+ cuda_ok(cudaMalloc(&d->conv_w, (size_t)conv_dim * convk * sizeof(float)), "dn conv weights") &&
+ cuda_ok(cudaMalloc(&d->norm_w, (size_t)vdim * sizeof(float)), "dn norm weights") &&
+ cuda_ok(cudaMalloc(&d->x, (size_t)hidden * sizeof(float)), "dn x") &&
+ cuda_ok(cudaMalloc(&d->qkvz, proj * sizeof(float)), "dn qkvz") &&
+ cuda_ok(cudaMalloc(&d->conv_out, (size_t)conv_dim * sizeof(float)), "dn conv out") &&
+ cuda_ok(cudaMalloc(&d->outr, (size_t)vh * vdim * sizeof(float)), "dn outr") &&
+ cuda_ok(cudaMalloc(&d->out, (size_t)hidden * sizeof(float)), "dn out") &&
+ cuda_ok(cudaMemcpy(d->conv_w, conv_w, (size_t)conv_dim * convk * sizeof(float), cudaMemcpyHostToDevice), "dn conv upload") &&
+ cuda_ok(cudaMemcpy(d->norm_w, norm_w, (size_t)vdim * sizeof(float), cudaMemcpyHostToDevice), "dn norm upload") &&
+ cuda_ok(cudaMemset(d->ring, 0, ring_b), "dn ring zero") && cuda_ok(cudaMemset(d->rec, 0, rec_b), "dn state zero");
+ if (!ok) { coli_cuda_dn_free(d); return NULL; }
+ return d;
+}
+extern "C" void coli_cuda_dn_free(ColiCudaDn *d) {
+ if (!d) return;
+ DeviceContext *ctx = find_ctx(d->device);
+ if (ctx) select_ctx(ctx);
+ float *bufs[] = { d->ring, d->rec, d->conv_w, d->norm_w, d->x, d->qkvz, d->conv_out, d->outr, d->out };
+ for (float *b : bufs) if (b) cudaFree(b);
+ std::free(d);
+}
+extern "C" int coli_cuda_dn_set_state(ColiCudaDn *d, const float *ring, const float *rec) {
+ if (!d) return 0;
+ if (!select_ctx(find_ctx(d->device))) return 0;
+ size_t ring_b = (size_t)d->conv_dim * (d->convk - 1) * sizeof(float), rec_b = (size_t)d->vh * d->kdim * d->vdim * sizeof(float);
+ int ok = ring ? cuda_ok(cudaMemcpy(d->ring, ring, ring_b, cudaMemcpyHostToDevice), "dn ring set")
+ : cuda_ok(cudaMemset(d->ring, 0, ring_b), "dn ring zero");
+ ok = ok && (rec ? cuda_ok(cudaMemcpy(d->rec, rec, rec_b, cudaMemcpyHostToDevice), "dn state set")
+ : cuda_ok(cudaMemset(d->rec, 0, rec_b), "dn state zero"));
+ return ok;
+}
+extern "C" int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec) {
+ if (!d || !ring || !rec) return 0;
+ if (!select_ctx(find_ctx(d->device))) return 0;
+ size_t ring_b = (size_t)d->conv_dim * (d->convk - 1) * sizeof(float), rec_b = (size_t)d->vh * d->kdim * d->vdim * sizeof(float);
+ return cuda_ok(cudaMemcpy(ring, d->ring, ring_b, cudaMemcpyDeviceToHost), "dn ring get") &&
+ cuda_ok(cudaMemcpy(rec, d->rec, rec_b, cudaMemcpyDeviceToHost), "dn state get");
+}
+extern "C" int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp,
+ const float *x, float *out, const float *egh, const float *beta) {
+ if (fault_injected()) return 0;
+ if (!d || !proj || !outp || !x || !out || !egh || !beta) return 0;
+ size_t proj_dim = (size_t)d->conv_dim + (size_t)d->vh * d->vdim, value_dim = (size_t)d->vh * d->vdim;
+ if (proj->fmt != 1 || proj->I != d->hidden || (size_t)proj->O != proj_dim || proj->device != d->device ||
+ outp->fmt != 1 || (size_t)outp->I != value_dim || outp->O != d->hidden || outp->device != d->device) return 0;
+ if (!select_ctx(find_ctx(d->device))) return 0;
+ DnGates g;
+ for (int h = 0; h < d->vh; h++) { g.egh[h] = egh[h]; g.beta[h] = beta[h]; }
+ if (!cuda_ok(cudaMemcpy(d->x, x, (size_t)d->hidden * sizeof(float), cudaMemcpyHostToDevice), "dn x upload")) return 0;
+ quant_matmul_launch(d->qkvz, d->x, proj->weights, proj->scales, 1, 1, d->hidden, (int)proj_dim,
+ row_bytes(1, d->hidden), proj->gs, proj->ng);
+ dn_conv_kernel<<<(unsigned)((d->conv_dim + 255) / 256), 256>>>(d->qkvz, d->ring, d->conv_w, d->conv_out, d->conv_dim, d->convk);
+ int threads = d->kdim > d->vdim ? d->kdim : d->vdim;
+ threads = ((threads + 31) / 32) * 32;
+ dn_head_kernel<<<(unsigned)d->vh, threads>>>(d->conv_out, d->qkvz + d->conv_dim, d->rec, d->outr, d->norm_w, g,
+ d->vh / d->vk, d->kdim, d->vdim, d->vk * d->kdim,
+ 1.f / sqrtf((float)d->kdim), d->eps);
+ quant_matmul_launch(d->out, d->outr, outp->weights, outp->scales, 1, 1, (int)value_dim, d->hidden,
+ row_bytes(1, (int)value_dim), outp->gs, outp->ng);
+ return cuda_ok(cudaGetLastError(), "dn step launch") &&
+ cuda_ok(cudaMemcpy(out, d->out, (size_t)d->hidden * sizeof(float), cudaMemcpyDeviceToHost), "dn out download");
+}
+
/* MXFP4 matmul, stateless. Separate from coli_cuda_matmul on purpose: that one
* takes scales as const float* and caches an uploaded tensor, while MXFP4
* scales are ue8m0 BYTES -- passing them through the float* parameter would
diff --git a/c/backend_cuda.h b/c/backend_cuda.h
index e58dbf95..9ecf6976 100644
--- a/c/backend_cuda.h
+++ b/c/backend_cuda.h
@@ -142,6 +142,25 @@ COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
const void *weights, const float *scales,
int fmt, int I, int O, int device);
+
+/* Gated delta layer on the device (the linear-attention layers of Qwen3.6/3.8,
+ * Kimi K3's KDA, GLM-5.3-Flash). One object per layer holds the short-conv ring
+ * [conv_dim x (convk-1)], the recurrent state [vh x kdim x vdim], the conv and
+ * norm weights and the scratch for one token. coli_cuda_dn_step runs a whole
+ * decode step on the card: x (host) -> in_proj GEMV -> causal conv + SiLU ->
+ * per head L2 norm, decay, delta rule, gated RMSNorm -> out_proj GEMV -> out
+ * (host). Only x and out cross the bus; the state never does. Gates (egh =
+ * exp(g) per value head, beta) arrive by value. All optional in the DLL
+ * loader: a backend without them leaves the layer on the CPU. */
+typedef struct ColiCudaDn ColiCudaDn;
+COLI_CUDA_DLLEXPORT ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim,
+ int conv_dim, int convk, int hidden,
+ const float *conv_w, const float *norm_w, float eps);
+COLI_CUDA_DLLEXPORT void coli_cuda_dn_free(ColiCudaDn *d);
+COLI_CUDA_DLLEXPORT int coli_cuda_dn_set_state(ColiCudaDn *d, const float *ring, const float *rec); /* NULL = zero */
+COLI_CUDA_DLLEXPORT int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec);
+COLI_CUDA_DLLEXPORT int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp,
+ const float *x, float *out, const float *egh, const float *beta);
#ifdef COLI_ANS
/* Experimental Linux-only GPU-resident entropy tier. The archive remains in
* VRAM and is decoded into per-device scratch immediately before a grouped
diff --git a/c/backend_loader.c b/c/backend_loader.c
index a3d4db20..f990135d 100644
--- a/c/backend_loader.c
+++ b/c/backend_loader.c
@@ -95,6 +95,11 @@ typedef int (*fn_attention_absorb)(ColiCudaTensor *kv_b, float *ctx,
typedef int (*fn_tensor_upload)(ColiCudaTensor **tensor, const void *weights,
const float *scales, int fmt, int I, int O, int device);
typedef int (*fn_tensor_upload_g)(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs);
+typedef ColiCudaDn *(*fn_dn_create)(int device, int vh, int vk, int kdim, int vdim, int conv_dim, int convk, int hidden, const float *conv_w, const float *norm_w, float eps);
+typedef void (*fn_dn_free)(ColiCudaDn *d);
+typedef int (*fn_dn_set_state)(ColiCudaDn *d, const float *ring, const float *rec);
+typedef int (*fn_dn_get_state)(ColiCudaDn *d, float *ring, float *rec);
+typedef int (*fn_dn_step)(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta);
typedef int (*fn_e8_set_grid)(const void *grid);
typedef int (*fn_fp8_set_lut)(const float *lut);
typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const float *x,
@@ -180,6 +185,7 @@ static struct {
fn_attention_absorb attention_absorb;
fn_tensor_upload tensor_upload;
fn_tensor_upload_g tensor_upload_g;
+ fn_dn_create dn_create; fn_dn_free dn_free; fn_dn_set_state dn_set_state; fn_dn_get_state dn_get_state; fn_dn_step dn_step; /* optional: gated delta layer on the device */
fn_e8_set_grid e8_set_grid;
fn_fp8_set_lut fp8_set_lut;
fn_matmul matmul;
@@ -1432,6 +1438,11 @@ static int coli_cuda_load(void){
RESOLVE(attention_absorb, fn_attention_absorb)
RESOLVE(tensor_upload, fn_tensor_upload)
RESOLVE(tensor_upload_g, fn_tensor_upload_g)
+ RESOLVE_OPT(dn_create, fn_dn_create)
+ RESOLVE_OPT(dn_free, fn_dn_free)
+ RESOLVE_OPT(dn_set_state, fn_dn_set_state)
+ RESOLVE_OPT(dn_get_state, fn_dn_get_state)
+ RESOLVE_OPT(dn_step, fn_dn_step)
RESOLVE_OPT(e8_set_grid, fn_e8_set_grid)
RESOLVE_OPT(fp8_set_lut, fn_fp8_set_lut)
RESOLVE(matmul, fn_matmul)
@@ -1640,6 +1651,19 @@ int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, const void *weights, cons
return g_cuda.tensor_upload_g(tensor, weights, scales, fmt, I, O, device, gs);
}
+/* Gated delta layer on the device: every entry point is optional, a DLL
+ * predating them answers NULL/0 and the engine keeps the layer on the CPU. */
+ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim, int conv_dim, int convk, int hidden, const float *conv_w, const float *norm_w, float eps){
+ if(!g_cuda.available || !g_cuda.dn_create || !g_cuda.dn_free || !g_cuda.dn_set_state || !g_cuda.dn_get_state || !g_cuda.dn_step) return NULL;
+ return g_cuda.dn_create(device, vh, vk, kdim, vdim, conv_dim, convk, hidden, conv_w, norm_w, eps);
+}
+void coli_cuda_dn_free(ColiCudaDn *d){ if(d && g_cuda.dn_free) g_cuda.dn_free(d); }
+int coli_cuda_dn_set_state(ColiCudaDn *d, const float *ring, const float *rec){ return d && g_cuda.dn_set_state ? g_cuda.dn_set_state(d, ring, rec) : 0; }
+int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec){ return d && g_cuda.dn_get_state ? g_cuda.dn_get_state(d, ring, rec) : 0; }
+int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta){
+ return d && g_cuda.dn_step ? g_cuda.dn_step(d, proj, outp, x, out, egh, beta) : 0;
+}
+
int coli_cuda_e8_set_grid(const void *grid){
if(!g_cuda.available || !g_cuda.e8_set_grid) return 0; /* fmt=6 stays CPU-side */
return g_cuda.e8_set_grid(grid);
diff --git a/c/tests/qwen36_fake_cuda.h b/c/tests/qwen36_fake_cuda.h
index 12980214..c1c5b9dd 100644
--- a/c/tests/qwen36_fake_cuda.h
+++ b/c/tests/qwen36_fake_cuda.h
@@ -131,4 +131,80 @@ int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x, const vo
return 1;
}
+
+/* Gated delta layer, host-side reference: the same arithmetic as the CUDA
+ * kernels (and as qwen36.c's deltanet()), so an engine test can put the layer
+ * on the fake tier and demand the same tokens as the CPU path. Matmuls go
+ * through the fake's fmt=1 compute when fake_dense_compute is on. */
+struct ColiCudaDn { int device, vh, vk, kdim, vdim, conv_dim, convk, hidden; float eps;
+ float *ring, *rec, *conv_w, *norm_w; };
+static int fake_dn_steps, fake_dn_fail;
+static void fake_gemv_i8(float *y, const float *x, const ColiCudaTensor *t) {
+ const int8_t *q = (const int8_t *)t->w;
+ for (int o = 0; o < t->O; o++) { float a = 0.f; const int8_t *w = q + (size_t)o * t->I;
+ for (int i = 0; i < t->I; i++) a += x[i] * (float)w[i]; y[o] = a * t->sc[o]; }
+}
+ColiCudaDn *coli_cuda_dn_create(int device, int vh, int vk, int kdim, int vdim, int conv_dim, int convk, int hidden,
+ const float *conv_w, const float *norm_w, float eps) {
+ if (fake_dn_fail || vh < 1 || vk < 1 || vh % vk || convk < 2 || conv_dim != 2 * vk * kdim + vh * vdim || !conv_w || !norm_w) return NULL;
+ ColiCudaDn *d = (ColiCudaDn *)calloc(1, sizeof *d);
+ d->device = device; d->vh = vh; d->vk = vk; d->kdim = kdim; d->vdim = vdim; d->conv_dim = conv_dim; d->convk = convk; d->hidden = hidden; d->eps = eps;
+ d->ring = (float *)calloc((size_t)conv_dim * (convk - 1), sizeof(float));
+ d->rec = (float *)calloc((size_t)vh * kdim * vdim, sizeof(float));
+ d->conv_w = (float *)malloc((size_t)conv_dim * convk * sizeof(float)); memcpy(d->conv_w, conv_w, (size_t)conv_dim * convk * sizeof(float));
+ d->norm_w = (float *)malloc((size_t)vdim * sizeof(float)); memcpy(d->norm_w, norm_w, (size_t)vdim * sizeof(float));
+ return d;
+}
+void coli_cuda_dn_free(ColiCudaDn *d) { if (!d) return; free(d->ring); free(d->rec); free(d->conv_w); free(d->norm_w); free(d); }
+int coli_cuda_dn_set_state(ColiCudaDn *d, const float *ring, const float *rec) {
+ if (!d) return 0;
+ size_t rn = (size_t)d->conv_dim * (d->convk - 1), sn = (size_t)d->vh * d->kdim * d->vdim;
+ if (ring) memcpy(d->ring, ring, rn * sizeof(float)); else memset(d->ring, 0, rn * sizeof(float));
+ if (rec) memcpy(d->rec, rec, sn * sizeof(float)); else memset(d->rec, 0, sn * sizeof(float));
+ return 1;
+}
+int coli_cuda_dn_get_state(ColiCudaDn *d, float *ring, float *rec) {
+ if (!d || !ring || !rec) return 0;
+ memcpy(ring, d->ring, (size_t)d->conv_dim * (d->convk - 1) * sizeof(float));
+ memcpy(rec, d->rec, (size_t)d->vh * d->kdim * d->vdim * sizeof(float));
+ return 1;
+}
+int coli_cuda_dn_step(ColiCudaDn *d, ColiCudaTensor *proj, ColiCudaTensor *outp, const float *x, float *out, const float *egh, const float *beta) {
+ if (!d || !proj || !outp || !x || !out || fake_dn_fail) return 0;
+ if (!fake_dense_compute || !proj->w || !outp->w) return 0; /* nothing to compute with: the engine keeps the CPU path */
+ int vh = d->vh, kdim = d->kdim, vdim = d->vdim, rep = vh / d->vk, key_dim_tot = d->vk * kdim, conv_dim = d->conv_dim, convk = d->convk;
+ size_t proj_dim = (size_t)conv_dim + (size_t)vh * vdim;
+ if (proj->fmt != 1 || proj->I != d->hidden || (size_t)proj->O != proj_dim || outp->fmt != 1 || (size_t)outp->I != (size_t)vh * vdim || outp->O != d->hidden) return 0;
+ float *qkvz = (float *)malloc(proj_dim * sizeof(float)), *conv_out = (float *)malloc((size_t)conv_dim * sizeof(float));
+ float *outr = (float *)malloc((size_t)vh * vdim * sizeof(float));
+ fake_gemv_i8(qkvz, x, proj);
+ const float *z = qkvz + conv_dim;
+ for (int cc = 0; cc < conv_dim; cc++) {
+ const float *w = d->conv_w + (size_t)cc * convk; float *rg = d->ring + (size_t)cc * (convk - 1);
+ float acc = 0.f; for (int kk = 0; kk < convk - 1; kk++) acc += w[kk] * rg[kk];
+ acc += w[convk - 1] * qkvz[cc]; conv_out[cc] = acc / (1.f + expf(-acc));
+ for (int kk = 0; kk < convk - 2; kk++) rg[kk] = rg[kk + 1];
+ rg[convk - 2] = qkvz[cc];
+ }
+ float scale = 1.f / sqrtf((float)kdim);
+ for (int h = 0; h < vh; h++) {
+ const float *qs = conv_out + (size_t)(h / rep) * kdim, *ks = conv_out + key_dim_tot + (size_t)(h / rep) * kdim, *vs = conv_out + 2 * (size_t)key_dim_tot + (size_t)h * vdim;
+ float q[256], k[256], delta[256], o[256];
+ double sq = 1e-6, sk = 1e-6;
+ for (int t = 0; t < kdim; t++) { sq += (double)qs[t] * qs[t]; sk += (double)ks[t] * ks[t]; }
+ double nq = sqrt(sq), nk = sqrt(sk);
+ for (int t = 0; t < kdim; t++) { q[t] = (float)((double)qs[t] / nq * scale); k[t] = (float)((double)ks[t] / nk); }
+ float *Sh = d->rec + (size_t)h * kdim * vdim;
+ for (int t = 0; t < vdim; t++) { float kvsum = 0.f; for (int kk = 0; kk < kdim; kk++) kvsum += k[kk] * (Sh[(size_t)kk * vdim + t] * egh[h]); delta[t] = (vs[t] - kvsum) * beta[h]; }
+ for (int t = 0; t < vdim; t++) { float acc = 0.f; for (int kk = 0; kk < kdim; kk++) { float s = Sh[(size_t)kk * vdim + t] * egh[h] + k[kk] * delta[t]; Sh[(size_t)kk * vdim + t] = s; acc += q[kk] * s; } o[t] = acc; }
+ double ms = 0; for (int t = 0; t < vdim; t++) ms += (double)o[t] * o[t];
+ float r = 1.f / sqrtf((float)(ms / vdim) + d->eps);
+ for (int t = 0; t < vdim; t++) { float zz = z[(size_t)h * vdim + t]; outr[(size_t)h * vdim + t] = (o[t] * r * d->norm_w[t]) * zz / (1.f + expf(-zz)); }
+ }
+ fake_gemv_i8(out, outr, outp);
+ free(qkvz); free(conv_out); free(outr);
+ fake_dn_steps++;
+ return 1;
+}
+
#endif /* QWEN36_FAKE_CUDA_H */
diff --git a/c/tests/test_backend_loader.py b/c/tests/test_backend_loader.py
index 52d15a7d..48269f16 100644
--- a/c/tests/test_backend_loader.py
+++ b/c/tests/test_backend_loader.py
@@ -918,7 +918,7 @@ class LoaderStubFixtureTest(unittest.TestCase):
cls.fixture = None
def test_abi_is_derived_from_the_loader_source(self):
- """47 mandatory + 9 optional, parsed from backend_loader.c.
+ """47 mandatory + 14 optional, parsed from backend_loader.c.
The counts are a deliberate tripwire: adding a RESOLVE to the loader
widens the ABI every Windows DLL must satisfy, and that should be a
@@ -929,8 +929,8 @@ class LoaderStubFixtureTest(unittest.TestCase):
"""
f = self.fixture
self.assertEqual(len(f.mandatory), 47)
- self.assertEqual(len(f.optional), 9) # +expert_mxfp4: optional Kimi SiTU pipeline
- self.assertEqual(len(f.exports), 56)
+ self.assertEqual(len(f.optional), 14) # +expert_mxfp4: optional Kimi SiTU pipeline; +dn_create/free/set_state/get_state/step
+ self.assertEqual(len(f.exports), 61)
self.assertEqual(len(f.exports), len(f.mandatory) + len(f.optional))
self.assertIn("coli_cuda_init", f.mandatory)
self.assertIn("coli_cuda_e8_set_grid", f.optional)
@@ -948,6 +948,11 @@ class LoaderStubFixtureTest(unittest.TestCase):
# would take the entire CUDA backend down over a sizing refinement.
self.assertIn("coli_cuda_tensor_vram", f.optional)
self.assertIn("coli_cuda_alloc_footprint", f.optional)
+ # dn_*: the DeltaNet layer on the device (conv ring, recurrence, gated
+ # norm). OPTIONAL: a DLL predating them leaves the layer on the CPU.
+ for name in ("coli_cuda_dn_create", "coli_cuda_dn_free", "coli_cuda_dn_set_state",
+ "coli_cuda_dn_get_state", "coli_cuda_dn_step"):
+ self.assertIn(name, f.optional)
def test_both_runtimes_exist_with_the_production_basename(self):
"""Same basename, different directories — the conflict precondition."""