Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: DeepSeek V4's expert activation in the expert batch

JustVugga4e8a2afdc8c
diff --git a/c/.gitignore b/c/.gitignore
index 98ecf217..54bcbbab 100644
--- a/c/.gitignore
+++ b/c/.gitignore
@@ -56,6 +56,7 @@ qwenimage.exe
shaders/attention_absorb.spv
shaders/qmatmul_gate_up.spv
shaders/expert_act.spv
+shaders/expert_act_v4.spv
shaders/rmsnorm.spv
# Optional XDNA test scaffolding that the *.o / *.dll globs above do not cover.
tests/xdna_exec_fixtures/
diff --git a/c/Makefile b/c/Makefile
index 2a91104c..1110a0a0 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -707,7 +707,8 @@ endif
# that does not call it links nothing from it.
VK_OBJ = backend_vulkan.o vk_tier.o
VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv \
- shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv shaders/expert_act.spv
+ shaders/qmatmul_gemm.spv shaders/qmatmul_coop.spv shaders/expert_act.spv \
+ shaders/expert_act_v4.spv
endif
# Optional AMD XDNA2 (Ryzen AI NPU) lane, Windows. Same opt-in shape as VK=1:
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index b4cd83e0..f45f1c36 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -2200,7 +2200,12 @@ static struct {
size_t *yoff; int nrows, cyoff;
unsigned long long batches, experts, rows, gemm_experts;
double dev_ms;
+ /* COLI_VK_ACT_SWIGLU_V4: expert_act_v4.spv between the activation and down, its
+ * row weights in the x buffer after each expert's rows (xb_v4_init) */
+ VkShaderModule sh_v4; VkDescriptorSetLayout dsl_v4; VkPipelineLayout pl_v4; VkPipeline p_v4;
+ VkDescriptorPool v4_pool; VkDescriptorSet v4_set;
} XB;
+struct PCV4 { int rows, n; };
static size_t xb_up(size_t v, size_t a) { return (v + a - 1) / a * a; }
@@ -2252,6 +2257,16 @@ static void xb_write_act_set(void) {
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC, .pBufferInfo = &bi[b]};
vkUpdateDescriptorSets(G.dev, 3, w, 0, NULL);
}
+/* expert_act_v4's set: the hidden rows in place, the weights from the x buffer */
+static void xb_write_v4_set(void) {
+ VkDescriptorBufferInfo bi[2] = {{XB.h.buf, 0, XB.h.region}, {XB.x.buf, 0, XB.x.region}};
+ VkWriteDescriptorSet w[2];
+ for (int b = 0; b < 2; b++)
+ w[b] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = XB.v4_set,
+ .dstBinding = (uint32_t)b, .descriptorCount = 1,
+ .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC, .pBufferInfo = &bi[b]};
+ vkUpdateDescriptorSets(G.dev, 2, w, 0, NULL);
+}
/* Make every scratch region hold `xr`/`ir`/`yr` bytes (offsets of a batch stay below
* its region; the buffer is twice that, so offset + window always fits). Grows by
@@ -2279,6 +2294,7 @@ static int xb_reserve(size_t xr, size_t ir, size_t yr) {
}
for (ColiVkExpert *e = XB.live; e; e = e->next) xb_write_sets(e);
xb_write_act_set();
+ if (XB.v4_set) xb_write_v4_set();
return 1;
}
@@ -2304,9 +2320,30 @@ static int xb_pipeline(VkShaderModule sh, VkPipelineLayout pl, const VkSpecializ
return 1;
}
+/* COLI_VK_ACT_SWIGLU_V4's pass (expert_act_v4.spv), made the first time that
+ * activation is asked for; 0 when its shader is missing. */
+static int xb_v4_init(void) {
+ if (XB.p_v4) return 1;
+ char path[1100];
+ derive_dir_file(G.spv_path, "expert_act_v4.spv", path, sizeof path);
+ if (!(XB.sh_v4 = load_spv(G.dev, path))) return 0;
+ if (!xb_layout(2, 3u, sizeof(struct PCV4), &XB.dsl_v4, &XB.pl_v4) ||
+ !xb_pipeline(XB.sh_v4, XB.pl_v4, NULL, &XB.p_v4)) return 0;
+ VkDescriptorPoolSize ps = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC, 2};
+ VkDescriptorPoolCreateInfo dp = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,
+ .maxSets = 1, .poolSizeCount = 1, .pPoolSizes = &ps};
+ VKCHECK(vkCreateDescriptorPool(G.dev, &dp, NULL, &XB.v4_pool), "xb v4 pool");
+ VkDescriptorSetAllocateInfo da = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
+ .descriptorPool = XB.v4_pool, .descriptorSetCount = 1, .pSetLayouts = &XB.dsl_v4};
+ VKCHECK(vkAllocateDescriptorSets(G.dev, &da, &XB.v4_set), "xb v4 set");
+ if (XB.h.buf && XB.x.buf) xb_write_v4_set();
+ return 1;
+}
+
int coli_vk_xb_init(int D, int I, int act, float limit, float a, float b) {
if (XB.ready) { /* the geometry stays; the activation rides the push constants */
if (XB.D != D || XB.I != I || XB.inflight) return 0;
+ if (act == COLI_VK_ACT_SWIGLU_V4 && !xb_v4_init()) return 0;
XB.act = act; XB.limit = limit > 0.f ? limit : 0.f; XB.a = a; XB.b = b;
return 1;
}
@@ -2358,6 +2395,7 @@ int coli_vk_xb_init(int D, int I, int act, float limit, float a, float b) {
/* a first scratch for 64 rows of 16 experts; grows with the first prefill */
if (!xb_reserve(64 * (size_t)D * 4 + 16 * XB.align, 64 * (size_t)I * 4 + 16 * XB.align,
64 * (size_t)D * 4 + 16 * XB.align)) return 0;
+ if (act == COLI_VK_ACT_SWIGLU_V4 && !xb_v4_init()) return 0;
XB.ready = 1;
return 1;
}
@@ -2421,10 +2459,13 @@ static void xb_barrier(VkCommandBuffer c) {
0, 1, &mb, 0, NULL, 0, NULL);
}
-int coli_vk_xb_issue(ColiVkExpert *const *ex, const int *rows, int count, const float *const *xrows) {
+static int xb_issue(ColiVkExpert *const *ex, const int *rows, int count, const float *const *xrows,
+ const float *wrows) {
if (!XB.ready || XB.inflight || count < 1) return 0;
const int D = XB.D, I = XB.I;
const size_t a = XB.align, dr = (size_t)D * 4, ir = (size_t)I * 4;
+ /* COLI_VK_ACT_SWIGLU_V4: each expert's row weights follow its x rows */
+ const int v4 = XB.act == COLI_VK_ACT_SWIGLU_V4 && XB.p_v4;
size_t *off = malloc((size_t)count * 3 * sizeof(size_t));
if (!off) return 0;
size_t xo = 0, io = 0, yo = 0; int total = 0;
@@ -2432,6 +2473,7 @@ int coli_vk_xb_issue(ColiVkExpert *const *ex, const int *rows, int count, const
if (!ex[c] || rows[c] < 1 || rows[c] > 65535) { free(off); return 0; }
off[3 * c] = xo; off[3 * c + 1] = io; off[3 * c + 2] = yo;
xo += xb_up((size_t)rows[c] * dr, a); io += xb_up((size_t)rows[c] * ir, a); yo += xb_up((size_t)rows[c] * dr, a);
+ if (v4) xo += xb_up((size_t)rows[c] * 4, a);
total += rows[c];
}
if (!xb_reserve(xo, io, yo)) { free(off); return 0; }
@@ -2446,6 +2488,7 @@ int coli_vk_xb_issue(ColiVkExpert *const *ex, const int *rows, int count, const
for (int r = 0; r < rows[c]; r++, j++) {
memcpy((uint8_t *)XB.x.ptr + off[3 * c] + (size_t)r * dr, xrows[j], dr);
XB.yoff[j] = off[3 * c + 2] + (size_t)r * dr;
+ if (v4) ((float *)((uint8_t *)XB.x.ptr + off[3 * c] + xb_up((size_t)rows[c] * dr, a)))[r] = wrows ? wrows[j] : 1.0f;
}
XB.nrows = total;
@@ -2511,6 +2554,17 @@ int coli_vk_xb_issue(ColiVkExpert *const *ex, const int *rows, int count, const
}
}
xb_barrier(cmd);
+ if (v4) { /* DeepSeek V4: route weight, bf16 and E4M3 per 128 on the hidden rows */
+ vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, XB.p_v4);
+ for (int c = 0; c < count; c++) {
+ uint32_t dyn[2] = {(uint32_t)off[3 * c + 1], (uint32_t)(off[3 * c] + xb_up((size_t)rows[c] * dr, a))};
+ vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, XB.pl_v4, 0, 1, &XB.v4_set, 2, dyn);
+ struct PCV4 pc = {rows[c], I};
+ vkCmdPushConstants(cmd, XB.pl_v4, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc);
+ vkCmdDispatch(cmd, (uint32_t)((I + 127) / 128), (uint32_t)rows[c], 1);
+ }
+ xb_barrier(cmd);
+ }
/* phase 3: down of every expert */
int bound = -2;
for (int c = 0; c < count; c++) {
@@ -2544,6 +2598,13 @@ int coli_vk_xb_issue(ColiVkExpert *const *ex, const int *rows, int count, const
XB.gemm_experts += (unsigned long long)ngemm;
return 1;
}
+int coli_vk_xb_issue(ColiVkExpert *const *ex, const int *rows, int count, const float *const *xrows) {
+ return xb_issue(ex, rows, count, xrows, NULL);
+}
+int coli_vk_xb_issue_w(ColiVkExpert *const *ex, const int *rows, int count, const float *const *xrows,
+ const float *wrows) {
+ return xb_issue(ex, rows, count, xrows, wrows);
+}
int coli_vk_xb_join(const float **yrows, double *device_ms) {
if (!XB.inflight) return 0;
@@ -2596,6 +2657,11 @@ static void xb_shutdown(void) {
if (XB.dsl6) vkDestroyDescriptorSetLayout(G.dev, XB.dsl6, NULL);
if (XB.dsl4) vkDestroyDescriptorSetLayout(G.dev, XB.dsl4, NULL);
if (XB.dsl3) vkDestroyDescriptorSetLayout(G.dev, XB.dsl3, NULL);
+ if (XB.p_v4) vkDestroyPipeline(G.dev, XB.p_v4, NULL);
+ if (XB.sh_v4) vkDestroyShaderModule(G.dev, XB.sh_v4, NULL);
+ if (XB.pl_v4) vkDestroyPipelineLayout(G.dev, XB.pl_v4, NULL);
+ if (XB.dsl_v4) vkDestroyDescriptorSetLayout(G.dev, XB.dsl_v4, NULL);
+ if (XB.v4_pool) vkDestroyDescriptorPool(G.dev, XB.v4_pool, NULL);
if (XB.qpool) vkDestroyQueryPool(G.dev, XB.qpool, NULL);
if (XB.fence) vkDestroyFence(G.dev, XB.fence, NULL);
if (XB.cpool) vkDestroyCommandPool(G.dev, XB.cpool, NULL);
@@ -3443,6 +3509,123 @@ static int run_tier_pool(void) {
return bad;
}
+/* COLI_VK_ACT_SWIGLU_V4 bit for bit. Experts whose sums are exact in f32 (f32 weights,
+ * one input that is a power of two, the others zero) and whose gate is at least 24, so
+ * that its sigmoid is exactly 1 on both sides: the device and the reference below then
+ * compute the same numbers through every rounding the activation makes -- gate and up
+ * to bf16, their product, the route weight and bf16 again, each block's power-of-two
+ * E4M3 scale (amax exactly 448 x 2^k among them) and every E4M3 rounding (ties to
+ * even, subnormal codes and their ties, values under half the smallest, the 448
+ * clamp), a 32-input tail block. Down is the identity, so a row of y IS the hidden
+ * row after the pass. Rows 1 and 3 take the GEMV route, 20 the GEMM. The reference is
+ * DeepSeek V4's CPU arithmetic (coli_bf16_round, coli_fp8_activation_qdq_ref), written
+ * out again here. */
+static float v4r_bf16(float v) {
+ uint32_t b; memcpy(&b, &v, 4);
+ if ((b & 0x7f800000u) != 0x7f800000u) b += 0x7fffu + ((b >> 16) & 1u);
+ b &= 0xffff0000u; memcpy(&v, &b, 4);
+ return v;
+}
+static float v4r_e4m3(float value) { /* encode (nearest even) then decode */
+ int neg = signbit(value) != 0;
+ float a = fabsf(value), r;
+ if (!a) return value;
+ if (a >= 448.0f) r = 448.0f;
+ else if (a < 0.015625f) {
+ float sc = a * 512.0f; unsigned q = (unsigned)sc; float fr = sc - (float)q;
+ if (fr > 0.5f || (fr == 0.5f && (q & 1))) q++;
+ r = ldexpf((float)q, -9);
+ } else {
+ uint32_t b; memcpy(&b, &a, 4);
+ int E = (int)((b >> 23) & 0xff) - 127;
+ uint32_t sig = 0x800000u | (b & 0x7fffffu), q = sig >> 20, rem = sig & 0xfffffu;
+ if (rem > 0x80000u || (rem == 0x80000u && (q & 1u))) q++;
+ r = ldexpf((float)q, E - 3);
+ }
+ return neg ? -r : r;
+}
+static void v4r_qdq(float *v, int n) {
+ for (int base = 0; base < n; base += 128) {
+ int cnt = n - base < 128 ? n - base : 128;
+ float mx = 0.0f;
+ for (int i = 0; i < cnt; i++) mx = fmaxf(mx, fabsf(v[base + i]));
+ mx = fmaxf(mx, 1e-4f);
+ int ex; float fr = frexpf(mx / 448.0f, &ex);
+ int e = fr == 0.5f ? ex - 1 : ex;
+ if (e < -127) e = -127;
+ if (e > 127) e = 127;
+ float scale = ldexpf(1.0f, e);
+ for (int i = 0; i < cnt; i++)
+ v[base + i] = v4r_e4m3(fmaxf(-448.0f, fminf(448.0f, v[base + i] / scale))) * scale;
+ }
+}
+static int run_xbatch_v4(void) {
+ enum { D = 160, I = 160, K = 3 }; /* one block of 128 and a tail of 32 */
+ static const int rows[K] = {1, 3, 20};
+ if (!coli_vk_xb_init(D, I, COLI_VK_ACT_SWIGLU_V4, 0.f, 0.f, 0.f)) {
+ printf("xbatch v4: no pass (expert_act_v4.spv missing beside the main shader?)\n"); return 1;
+ }
+ /* special values of the hidden row (x 2^-4: the block's amax 28 makes its scale
+ * 2^-4): 448, ties of the normal and the subnormal codes, under half the smallest */
+ static const float tval[] = {448.0f, 1.0625f, 1.1875f, -2.125f, 15.5f, -15.5f, 447.0f, 440.0f,
+ 3.5f / 512, 2.5f / 512, -1.5f / 512, 0.4f / 512, 0.75f / 512, 0.0f, 300.0f, -0.5f};
+ const int nt = (int)(sizeof tval / sizeof *tval);
+ float *gw[K], *uw[K], *dw = calloc((size_t)D * I, sizeof(float));
+ ColiVkExpert *ex[K];
+ int bad = 0;
+ for (int d = 0; d < D; d++) dw[(size_t)d * I + d] = 1.0f;
+ for (int c = 0; c < K; c++) {
+ gw[c] = calloc((size_t)I * D, sizeof(float)); uw[c] = calloc((size_t)I * D, sizeof(float));
+ for (int o = 0; o < I; o++) {
+ /* column 0 only: g = 24..63 with few bits, u random -- or, in expert 0's
+ * first block, g = 32 and u = t / 512 (h = t / 16) for the special values
+ * and the rest under them, so the block's amax is 28 */
+ float g = 24.0f + (float)(rand() % 40), u = ((float)rand() / RAND_MAX - 0.5f) * ldexpf(1.0f, rand() % 12 - 8);
+ if (c == 0 && o < 128) u = ((float)rand() / RAND_MAX - 0.5f) * ldexpf(1.0f, -(rand() % 12) - 6);
+ if (c == 0 && o < nt) { g = 32.0f; u = tval[o] / 512.0f; }
+ gw[c][(size_t)o * D] = g; uw[c][(size_t)o * D] = u;
+ }
+ ColiVkTensor *t[3]; const float *src[3] = {gw[c], uw[c], dw};
+ for (int k = 0; k < 3; k++) {
+ uint8_t *rws; size_t stride; float *sc;
+ if (!coli_vk_tier_tensor(&t[k], 10, k < 2 ? D : I, k < 2 ? I : D, 0, &rws, &stride, &sc)) {
+ printf("xbatch v4: no tensor\n"); return 1;
+ }
+ for (int o = 0; o < (k < 2 ? I : D); o++) memcpy(rws + (size_t)o * stride, src[k] + (size_t)o * D, (size_t)D * 4);
+ sc[0] = 1.0f;
+ }
+ if (!(ex[c] = coli_vk_xb_expert(t[0], t[1], t[2]))) { printf("xbatch v4: no expert\n"); return 1; }
+ }
+ int total = 0; for (int c = 0; c < K; c++) total += rows[c];
+ float *x = calloc((size_t)total * D, sizeof(float)), *w = malloc((size_t)total * sizeof(float));
+ const float **xr = malloc((size_t)total * sizeof(*xr)), **yr = malloc((size_t)total * sizeof(*yr));
+ static const float xs[4] = {1.0f, 2.0f, 0.5f, 1.0f}, ws[5] = {1.0f, 0.75f, 1.5f, 0.3f, 1.0f};
+ for (int j = 0; j < total; j++) { x[(size_t)j * D] = j == 0 ? 1.0f : xs[j % 4]; w[j] = j == 0 ? 1.0f : ws[j % 5]; xr[j] = x + (size_t)j * D; }
+ int mism = 0, sub = 0; double dms = 0;
+ if (!coli_vk_xb_issue_w(ex, rows, K, xr, w) || !coli_vk_xb_join(yr, &dms)) { printf("xbatch v4: issue/join failed\n"); bad = 1; }
+ float h[I];
+ for (int c = 0, j = 0; !bad && c < K; c++)
+ for (int r = 0; r < rows[c]; r++, j++) {
+ for (int o = 0; o < I; o++) {
+ float g = v4r_bf16(x[(size_t)j * D] * gw[c][(size_t)o * D]), u = v4r_bf16(x[(size_t)j * D] * uw[c][(size_t)o * D]);
+ float sg = g >= 0.0f ? 1.0f / (1.0f + expf(-g)) : expf(g) / (1.0f + expf(g));
+ h[o] = v4r_bf16(g * sg * u * w[j]);
+ }
+ v4r_qdq(h, I);
+ for (int o = 0; o < I; o++) {
+ if (yr[j][o] != h[o]) { if (mism < 5) printf(" v4 mismatch: expert %d row %d [%d]: device %.9g, reference %.9g\n", c, r, o, yr[j][o], h[o]); mism++; }
+ if (c == 0 && j == 0 && o < nt) sub += h[o] != 0.0f && fabsf(h[o]) < ldexpf(1.0f, -10); /* subnormal codes */
+ }
+ }
+ printf("xbatch v4 D=%d I=%d, rows 1/3/20 (GEMM from %d) | %d of %d values differ from the CPU's arithmetic | device %.3f ms\n",
+ D, I, XB.gemm_rows, mism, total * I, dms);
+ if (mism || !sub) bad = 1;
+ for (int c = 0; c < K; c++) { coli_vk_xb_expert_free(ex[c]); free(gw[c]); free(uw[c]); }
+ free(dw); free(x); free(w); free(xr); free(yr);
+ xb_shutdown();
+ return bad;
+}
+
/* Every format through the batch, both routes, the two activations, a hidden size
* past the gate_up staging array, odd widths and a down format other than gate/up's. */
static int run_xbatch_all(void) {
@@ -3469,6 +3652,7 @@ static int run_xbatch_all(void) {
if (!coli_vk_xb_init(6400, 64, COLI_VK_ACT_SWIGLU, 0.f, 0.f, 0.f)) return 1;
bad |= run_xbatch(4, 4, 64, 6400, 64, COLI_VK_ACT_SWIGLU, 0.f);
xb_shutdown();
+ bad |= run_xbatch_v4(); /* DeepSeek V4's activation, bit for bit */
return bad;
}
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index d7f0f315..e60dfc31 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -171,6 +171,13 @@ size_t coli_vk_tensor_scale_count(int fmt, int I, int O, int gs);
* Threading: engine thread only, except where noted. */
#define COLI_VK_ACT_SWIGLU 0
#define COLI_VK_ACT_SITU 1
+/* DeepSeek V4's expert, with the roundings its CPU kernel makes: gate and up rounded
+ * to bf16, the clamped SwiGLU, times the row's route weight (coli_vk_xb_issue_w) and
+ * rounded to bf16, then quantized to E4M3 and back with one power-of-two scale per
+ * 128 inputs before down (expert_act_v4.spv beside the main shader). The rows given
+ * are already E4M3-rounded by the caller, as the CPU kernel rounds x; down's output
+ * comes back in f32 for the caller's own bf16 rounding. */
+#define COLI_VK_ACT_SWIGLU_V4 2
typedef struct ColiVkExpert ColiVkExpert;
int coli_vk_xb_init(int D, int I, int act, float limit, float a, float b); /* again: same D, I, new act */
int coli_vk_xb_ready(void);
@@ -185,6 +192,10 @@ void coli_vk_xb_expert_free(ColiVkExpert *e);
* sum(rows) pointers to D floats, expert by expert. 0 = nothing was submitted (the
* caller computes those experts itself). One batch in flight at a time. */
int coli_vk_xb_issue(ColiVkExpert *const *ex, const int *rows, int count, const float *const *xrows);
+/* The same with one weight per input row (same order as xrows, NULL = 1), which
+ * COLI_VK_ACT_SWIGLU_V4 applies before down; the other activations ignore it. */
+int coli_vk_xb_issue_w(ColiVkExpert *const *ex, const int *rows, int count, const float *const *xrows,
+ const float *wrows);
/* Wait for it: yrows[j] points at the D outputs of input row j (same order), valid
* until the next issue; *device_ms is its device time when timestamps exist (else 0).
* 0 = the batch failed (device lost): the caller computes those rows itself. */
diff --git a/c/shaders/expert_act.comp b/c/shaders/expert_act.comp
index 2e4ddb94..a4669870 100644
--- a/c/shaders/expert_act.comp
+++ b/c/shaders/expert_act.comp
@@ -6,6 +6,8 @@
// either way differs only in the projections' summation order:
// 0 SwiGLU silu(g) * u, gate clamped from above and up to [-limit, limit] when limit > 0
// 1 SiTU-GLU a*tanh(g/a)*sigmoid(g) * b*tanh(u/b)
+// 2 DeepSeek V4: gate and up rounded to bf16, then 0's clamps and g * sigmoid(g) * u
+// in its CPU's form (act_v4, as in qmatmul_gate_up.comp)
layout(local_size_x = 256) in;
layout(std430, binding = 0) readonly buffer G { float gate[]; };
@@ -19,12 +21,27 @@ layout(push_constant) uniform P {
float a, b; // act 1's two bounds
} p;
+// act 2, the same functions as qmatmul_gate_up.comp's
+float bf16r(float v) {
+ uint b = floatBitsToUint(v);
+ if ((b & 0x7f800000u) != 0x7f800000u) b += 0x7fffu + ((b >> 16) & 1u);
+ return uintBitsToFloat(b & 0xffff0000u);
+}
+float act_v4(float g, float u, float limit) {
+ g = bf16r(g); u = bf16r(u);
+ if (limit > 0.0) { g = min(g, limit); u = max(-limit, min(u, limit)); }
+ float s = g >= 0.0 ? 1.0 / (1.0 + exp(-g)) : exp(g) / (1.0 + exp(g));
+ return g * s * u;
+}
+
void main() {
int i = int(gl_GlobalInvocationID.x) + int(gl_GlobalInvocationID.y) * int(gl_NumWorkGroups.x) * 256;
if (i >= p.n) return;
float gt = gate[i], ut = up[i], h;
if (p.act == 1) {
h = p.a * tanh(gt / p.a) * (1.0 / (1.0 + exp(-gt))) * (p.b * tanh(ut / p.b));
+ } else if (p.act == 2) {
+ h = act_v4(gt, ut, p.limit);
} else {
if (p.limit > 0.0) {
if (gt > p.limit) gt = p.limit;
diff --git a/c/shaders/expert_act_v4.comp b/c/shaders/expert_act_v4.comp
new file mode 100644
index 00000000..36e48b3f
--- /dev/null
+++ b/c/shaders/expert_act_v4.comp
@@ -0,0 +1,75 @@
+#version 450
+// DeepSeek V4's routed expert between its activation (act 2 of qmatmul_gate_up.comp
+// and expert_act.comp) and down, in place over one expert's hidden rows:
+// hid[r][i] = e4m3(bf16(hid[r][i] * w[r]))
+// with w[r] the row's route weight, bf16 rounding to nearest even, and e4m3 the
+// quantize-dequantize the engine's fp4 kernel applies to down's input on the CPU
+// (coli_fp8_activation_qdq_ref, blocks of 128): per block a power-of-two scale
+// 2^e with e the smallest integer for which max(amax, 1e-4) <= 448 * 2^e, each value
+// divided by it, clamped to [-448, 448], rounded to E4M3 (nearest even, subnormals
+// in steps of 2^-9) and multiplied back. Every step is exact in f32 or rounds once
+// as the CPU does, so a row equals the CPU's whenever its activation does.
+// One workgroup per (block, row); a block past the end of a row is partial.
+layout(local_size_x = 128) in;
+
+layout(std430, binding = 0) buffer H { float hid[]; };
+layout(std430, binding = 1) readonly buffer W { float rw[]; };
+
+layout(push_constant) uniform P {
+ int rows;
+ int n; // elements per row: moe_inter
+} p;
+
+shared float amax[128];
+
+float bf16r(float v) {
+ uint b = floatBitsToUint(v);
+ if ((b & 0x7f800000u) != 0x7f800000u) b += 0x7fffu + ((b >> 16) & 1u);
+ return uintBitsToFloat(b & 0xffff0000u);
+}
+float pow2(int k) { return uintBitsToFloat(uint(k + 127) << 23); } // 2^k, -126 <= k <= 127
+
+// |x| <= 448: the E4M3 value nearest to x (ties to even), as coli_e4m3fn_encode
+// followed by coli_e4m3fn_decode
+float e4m3r(float x) {
+ float a = abs(x), r;
+ if (a == 0.0) return x;
+ if (a >= 448.0) r = 448.0;
+ else if (a < 0.015625) { // below 2^-6: subnormal codes, steps of 2^-9
+ float s = a * 512.0, f = floor(s), fr = s - f;
+ if (fr > 0.5 || (fr == 0.5 && (int(f) & 1) == 1)) f += 1.0;
+ r = f * 0.001953125;
+ } else { // 1.mmm x 2^E, three mantissa bits
+ uint b = floatBitsToUint(a);
+ int E = int((b >> 23) & 0xffu) - 127;
+ uint sig = 0x800000u | (b & 0x7fffffu), q = sig >> 20, rem = sig & 0xfffffu;
+ if (rem > 0x80000u || (rem == 0x80000u && (q & 1u) == 1u)) q++;
+ r = float(q) * pow2(E - 3);
+ }
+ return x < 0.0 ? -r : r;
+}
+
+void main() {
+ int t = int(gl_LocalInvocationID.x), r = int(gl_WorkGroupID.y);
+ int i = int(gl_WorkGroupID.x) * 128 + t;
+ bool live = r < p.rows && i < p.n;
+ uint at = uint(r) * uint(p.n) + uint(max(i, 0));
+ float v = live ? bf16r(hid[at] * rw[r]) : 0.0;
+ amax[t] = abs(v);
+ barrier();
+ for (int s = 64; s > 0; s >>= 1) {
+ if (t < s) amax[t] = max(amax[t], amax[t + s]);
+ barrier();
+ }
+ // m = (1 + man / 2^23) * 2^E: m <= 448 * 2^e, 448 = 1.75 * 2^8, holds from
+ // e = E - 8 when the mantissa is at most 1.75, else from e = E - 7. (The CPU
+ // divides by 448 and takes the ceiling of the log; its quotient is correctly
+ // rounded, so the two agree, a power of two included.)
+ float m = max(amax[0], 1e-4);
+ uint mb = floatBitsToUint(m);
+ int E = int((mb >> 23) & 0xffu) - 127;
+ int e = (mb & 0x7fffffu) <= 0x600000u ? E - 8 : E - 7;
+ e = clamp(e, -126, 126);
+ float x = clamp(v * pow2(-e), -448.0, 448.0);
+ if (live) hid[at] = e4m3r(x) * pow2(e);
+}
diff --git a/c/shaders/qmatmul_gate_up.comp b/c/shaders/qmatmul_gate_up.comp
index 091f5192..3a0a6b34 100644
--- a/c/shaders/qmatmul_gate_up.comp
+++ b/c/shaders/qmatmul_gate_up.comp
@@ -15,6 +15,9 @@
// 0 SwiGLU silu(g) * u; with limit > 0 the gate is clamped from above at limit
// and up to [-limit, limit] first (swiglu_limit; 0 = unclamped)
// 1 SiTU-GLU a*tanh(g/a)*sigmoid(g) * b*tanh(u/b) (Kimi K3: a = 4, b = 25)
+// 2 DeepSeek V4: gate and up rounded to bf16 first, then act 0's clamps and
+// g * sigmoid(g) * u as its CPU computes it (act_v4 below); the route weight,
+// bf16 and E4M3 rounding that follow run in expert_act_v4.comp
#extension GL_KHR_shader_subgroup_arithmetic : require
layout(local_size_x = 256) in;
@@ -56,6 +59,20 @@ float e4m3(uint w, int l) {
return (b & 0x80u) != 0u ? -v : v;
}
+// act 2: round to bf16, nearest even (the CPU's coli_bf16_round), and DeepSeek V4's
+// SwiGLU in its CPU form (coli_v4_swiglu after coli_bf16_round_array of gate and up)
+float bf16r(float v) {
+ uint b = floatBitsToUint(v);
+ if ((b & 0x7f800000u) != 0x7f800000u) b += 0x7fffu + ((b >> 16) & 1u);
+ return uintBitsToFloat(b & 0xffff0000u);
+}
+float act_v4(float g, float u, float limit) {
+ g = bf16r(g); u = bf16r(u);
+ if (limit > 0.0) { g = min(g, limit); u = max(-limit, min(u, limit)); }
+ float s = g >= 0.0 ? 1.0 / (1.0 + exp(-g)) : exp(g) / (1.0 + exp(g));
+ return g * s * u;
+}
+
void main() {
int s = int(gl_WorkGroupID.y);
int tid = int(gl_LocalInvocationID.x);
@@ -164,6 +181,8 @@ void main() {
float h;
if (p.act == 1) {
h = p.a * tanh(gt / p.a) * (1.0 / (1.0 + exp(-gt))) * (p.b * tanh(ut / p.b));
+ } else if (p.act == 2) {
+ h = act_v4(gt, ut, p.limit);
} else {
if (p.limit > 0.0) {
if (gt > p.limit) gt = p.limit;