Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: DeepSeek V4's roundings on the chain

JustVugg32185798b4e4
diff --git a/c/shaders/chain_dsv4.comp b/c/shaders/chain_dsv4.comp
index 1e210b19..f6d6c04a 100644
--- a/c/shaders/chain_dsv4.comp
+++ b/c/shaders/chain_dsv4.comp
@@ -1,7 +1,7 @@
#version 450
// DeepSeek V4.1 Flash's (and DeepSeek V4's) attention on the dense chain (vk_chain.c,
// chain_dsv4; vk_chain.h: vkc_dsv4_*). The model is MQA over one KV row per position:
-// a sliding window of raw rows plus compressed rows a DSA indexer picks. Seven modes:
+// a sliding window of raw rows plus compressed rows a DSA indexer picks. Nine modes:
//
// mode 0, sparse attention with a sink (sparse_attn.h, coli_sparse_attend): one
// workgroup per (head h, row s). The row's list is list[l_off + s*l_row + j], j < cnt:
@@ -46,6 +46,27 @@
// s): x = the stream (x_off + s*x_row + c*D), key = kv[s][c*D..], value = kv[s][H*D..];
// r = 1/sqrt(mean(x^2) + eps) * 1/sqrt(mean(key^2) + eps), t = sum(x * qw * kw * key)
// * r / sqrt(D), g = sigmoid(sign(t) * sqrt(max(|t|, 1e-6))); x += g * value.
+// mode 7, DeepSeek V4's roundings over segments: segment g (row g / per_row, index
+// j = g % per_row) is len floats at x_off + row*x_row + j*x_seg, written to y at
+// y_off + row*y_row + j*y_seg (in place when x is y). Each kind as the engine's C:
+// kind 0, bf16 (coli_bf16_round). One thread per value.
+// kind 1, E4M3 per block of `block` values (coli_fp8_activation_qdq_ref): amax floored
+// at 1e-4, the scale 2^e with e the smallest for which amax <= 448 * 2^e, each value
+// times 2^-e clamped to +-448, rounded to E4M3 (nearest even, steps of 2^-9 below
+// 2^-6) and times 2^e back. One workgroup per (segment, block), block <= 256.
+// kind 2, E2M1 per block (coli_fp4_activation_qdq_ref): amax floored at 6 * 2^-126, the
+// scale 2^e smallest for which amax <= 6 * 2^e, each value times 2^-e clamped to +-6
+// and rounded to the nearest of 0, 0.5, 1, 1.5, 2, 3, 4, 6 (a tie to the smaller,
+// zero as +0), times 2^e. One workgroup per (segment, block), block <= 256.
+// kind 3, the Hadamard transform (coli_hadamard_bf16_ref): butterflies of width 1, 2,
+// 4, ... in place in the CPU's order, then each value times hscale (the host's
+// 1 / sqrtf(len)) to bf16. One workgroup per segment, len a power of two <= 4096.
+// flags 1 (kinds 1 and 2): the result to bf16 after. Every step is exact in f32 or
+// rounds once as the CPU does, so the result is the CPU's bit for bit.
+// mode 8, DeepSeek V4's SwiGLU between the shared expert's roundings (coli_v4_swiglu):
+// over n values g = bf16(a[a_off + i]), u = bf16(b[b_off + i]); with lim > 0,
+// g = min(g, lim) and u = clamp(u, -lim, lim); y[y_off + i] = bf16(g * sigmoid(g) * u),
+// the sigmoid in the CPU's stable form.
// Offsets and strides in floats (ints for lists and masks).
#extension GL_KHR_shader_subgroup_basic : enable
layout(local_size_x = 256) in;
@@ -394,6 +415,108 @@ void engram() {
for (int i = tid; i < D; i += 256) y5[xb + i] = y5[xb + i] + g * b0[vb + i];
}
+// ---- mode 7 ------------------------------------------------------------------------
+float pow2(int k) { return uintBitsToFloat(uint(k + 127) << 23); } // 2^k, -126 <= k <= 127
+// |x| <= 448: the nearest E4M3 value (ties to even), coli_e4m3fn_encode then _decode
+float e4m3r(float x) {
+ float a = abs(x), r;
+ if (a == 0.0) return x;
+ if (a >= 448.0) r = 448.0;
+ else if (a < 0.015625) { // below 2^-6: subnormal codes, steps of 2^-9
+ float s = a * 512.0, f = floor(s), fr = s - f;
+ if (fr > 0.5 || (fr == 0.5 && (int(f) & 1) == 1)) f += 1.0;
+ r = f * 0.001953125;
+ } else { // 1.mmm x 2^E, three mantissa bits
+ uint b = floatBitsToUint(a);
+ int E = int((b >> 23) & 0xffu) - 127;
+ uint sg = 0x800000u | (b & 0x7fffffu), q = sg >> 20, rem = sg & 0xfffffu;
+ if (rem > 0x80000u || (rem == 0x80000u && (q & 1u) == 1u)) q++;
+ r = float(q) * pow2(E - 3);
+ }
+ return x < 0.0 ? -r : r;
+}
+// |x| <= 6: the nearest E2M1 value, a tie to the smaller magnitude, zero as +0 (the CPU
+// scans the codes in order and keeps the first of two equal distances)
+float e2m1r(float x) {
+ float a = abs(x), r;
+ if (a <= 0.25) return 0.0;
+ if (a <= 0.75) r = 0.5;
+ else if (a <= 1.25) r = 1.0;
+ else if (a <= 1.75) r = 1.5;
+ else if (a <= 2.5) r = 2.0;
+ else if (a <= 3.5) r = 3.0;
+ else if (a <= 5.0) r = 4.0;
+ else r = 6.0;
+ return x < 0.0 ? -r : r;
+}
+float rd_in(int i) { return p.v[13] != 0 ? y5[i] : b0[i]; }
+void roundings() {
+ int g = int(gl_WorkGroupID.x + gl_WorkGroupID.y * gl_NumWorkGroups.x), tid = int(gl_LocalInvocationID.x);
+ int kind = p.v[1], nseg = p.v[2], per_row = p.v[3], len = p.v[4], blk = p.v[5], flags = p.v[6];
+ if (kind == 0) {
+ int t = g * 256 + tid;
+ if (t >= nseg * len) return;
+ int sg = t / len, i = t - sg * len, row = sg / per_row, j = sg - row * per_row;
+ float v = rd_in(p.v[7] + row * p.v[8] + j * p.v[9] + i);
+ y5[p.v[10] + row * p.v[11] + j * p.v[12] + i] = bf16r(v);
+ return;
+ }
+ if (kind == 3) { // one workgroup per segment
+ if (g >= nseg) return;
+ int row = g / per_row, j = g - row * per_row;
+ int xb = p.v[7] + row * p.v[8] + j * p.v[9], yb = p.v[10] + row * p.v[11] + j * p.v[12];
+ for (int i = tid; i < len; i += 256) sm[i] = rd_in(xb + i);
+ barrier();
+ for (int w = 1; w < len; w *= 2) {
+ for (int k = tid; k < len / 2; k += 256) {
+ int base = (k / w) * 2 * w, i = k - (k / w) * w;
+ float l = sm[base + i], r = sm[base + w + i];
+ sm[base + i] = l + r;
+ sm[base + w + i] = l - r;
+ }
+ barrier();
+ }
+ float hs = intBitsToFloat(p.v[14]);
+ for (int i = tid; i < len; i += 256) y5[yb + i] = bf16r(sm[i] * hs);
+ return;
+ }
+ // kinds 1 and 2: one workgroup per (segment, block)
+ int nblk = (len + blk - 1) / blk, sg = g / nblk, bk = g - sg * nblk;
+ if (sg >= nseg) return;
+ int row = sg / per_row, j = sg - row * per_row, i = bk * blk + tid;
+ bool live = tid < blk && i < len;
+ int xa = p.v[7] + row * p.v[8] + j * p.v[9] + i, ya = p.v[10] + row * p.v[11] + j * p.v[12] + i;
+ float v = live ? rd_in(xa) : 0.0;
+ red[tid] = abs(v);
+ barrier();
+ for (int off = 128; off > 0; off >>= 1) {
+ if (tid < off) red[tid] = max(red[tid], red[tid + off]);
+ barrier();
+ }
+ // m = (1 + man / 2^23) * 2^E. E4M3: 448 = 1.75 * 2^8, so m <= 448 * 2^e holds from
+ // e = E - 8 when the mantissa is at most 1.75, else from E - 7; E2M1: 6 = 1.5 * 2^2,
+ // from E - 2 when it is at most 1.5, else from E - 1. (The CPU takes the ceiling of
+ // log2 of m / 448 or m / 6, a correctly rounded quotient: the two agree.)
+ float m = kind == 1 ? max(red[0], 1.0e-4) : max(red[0], 6.0 * pow2(-126));
+ uint mb = floatBitsToUint(m), man = mb & 0x7fffffu;
+ int E = int((mb >> 23) & 0xffu) - 127;
+ int e = kind == 1 ? (man <= 0x600000u ? E - 8 : E - 7) : (man <= 0x400000u ? E - 2 : E - 1);
+ e = clamp(e, -126, 126);
+ float r = kind == 1 ? e4m3r(clamp(v * pow2(-e), -448.0, 448.0)) * pow2(e)
+ : e2m1r(clamp(v * pow2(-e), -6.0, 6.0)) * pow2(e);
+ if ((flags & 1) != 0) r = bf16r(r);
+ if (live) y5[ya] = r;
+}
+
+// ---- mode 8 ------------------------------------------------------------------------
+void swiglu_v4() {
+ int t = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 256u);
+ if (t >= p.v[1]) return;
+ float g = bf16r(b0[p.v[2] + t]), u = bf16r(b1[p.v[3] + t]), lim = intBitsToFloat(p.v[5]);
+ if (lim > 0.0) { g = min(g, lim); u = max(-lim, min(u, lim)); }
+ y5[p.v[4] + t] = bf16r(g * sig(g) * u);
+}
+
void main() {
int m = MODE;
if (m == 0) attend();
@@ -403,4 +526,6 @@ void main() {
else if (m == 4) candidates();
else if (m == 5) topk();
else if (m == 6) engram();
+ else if (m == 7) roundings();
+ else if (m == 8) swiglu_v4();
}
diff --git a/c/shaders/chain_dsv4.spv b/c/shaders/chain_dsv4.spv
index cd2546b6..b9386e22 100644
Binary files a/c/shaders/chain_dsv4.spv and b/c/shaders/chain_dsv4.spv differ
diff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.c
index 2b4a98bc..495730d9 100644
--- a/c/tests/test_vk_chain.c
+++ b/c/tests/test_vk_chain.c
@@ -36,6 +36,8 @@
* window and compressed rows (V4's bf16 roundings too), interleaved RoPE
* both ways, the compressor's ring (V4's overlapping form), the indexer's
* scores, candidate blocks and top-k slot for slot, the engram gate
+ * dsv4 rounding DeepSeek V4's roundings (deepseek_v4.c): bf16, E4M3 and E2M1 per block,
+ * the Hadamard transform, bit for bit; its SwiGLU within one bf16 step
*
* make vk-chain-check VK=1 (VK_ICD_FILENAMES=.../lvp_icd.json for Lavapipe) */
#include <stdio.h>
@@ -1245,6 +1247,179 @@ static void test_ds_engram(int S, int H, int D) {
free(kv); free(x); free(prm); free(ref); free(got);
}
+/* ---- DeepSeek V4's roundings (chain_dsv4.comp modes 7, 8) ------------------------------
+ * References transcribed from deepseek_v4.c (COLI_V4_UNIT_NATIVE_QUANT, the MATH unit's
+ * coli_v4_swiglu): the device must give their bits. */
+static float ds_e4m3_decode(uint8_t v) {
+ int sign = v >> 7, exponent = (v >> 3) & 15, mantissa = v & 7;
+ if (exponent == 15 && mantissa == 7) return NAN;
+ float n = !exponent ? ldexpf((float)mantissa, -9) : ldexpf(1.0f + (float)mantissa / 8.0f, exponent - 7);
+ return sign ? -n : n;
+}
+static uint8_t ds_e4m3_encode(float value) {
+ if (isnan(value)) return 0x7f;
+ int negative = signbit(value) != 0;
+ float magnitude = fabsf(value);
+ if (!magnitude) return negative ? 0x80 : 0;
+ if (magnitude >= 448.0f) return (uint8_t)((negative ? 0x80 : 0) | 0x7e);
+ uint8_t best;
+ if (magnitude < 0.015625f) {
+ float scaled = magnitude * 512.0f;
+ uint8_t rounded = (uint8_t)scaled;
+ float fraction = scaled - rounded;
+ if (fraction > 0.5f || (fraction == 0.5f && (rounded & 1))) rounded++;
+ best = rounded;
+ } else {
+ uint32_t bits; memcpy(&bits, &magnitude, 4);
+ int exponent = (int)((bits >> 23) & 0xff) - 127;
+ uint32_t significand = 0x800000u | (bits & 0x7fffffu), rounded = significand >> 20, remainder = significand & 0xfffffu;
+ if (remainder > 0x80000u || (remainder == 0x80000u && (rounded & 1u))) rounded++;
+ if (rounded == 16u) { rounded = 8u; exponent++; }
+ best = (uint8_t)((exponent + 7) * 8 + (int)rounded - 8);
+ }
+ return (uint8_t)(best | (negative ? 0x80 : 0));
+}
+static int ds_ceil_log2(float value) { int e; float f = frexpf(value, &e); return f == 0.5f ? e - 1 : e; }
+static void ds_fp8_qdq(float *out, const float *in, size_t n, size_t block) {
+ for (size_t base = 0; base < n; base += block) {
+ size_t count = n - base < block ? n - base : block;
+ float maximum = 0.0f;
+ for (size_t i = 0; i < count; i++) maximum = fmaxf(maximum, fabsf(in[base + i]));
+ maximum = fmaxf(maximum, 1e-4f);
+ int e = ds_ceil_log2(maximum / 448.0f);
+ if (e < -127) e = -127;
+ if (e > 127) e = 127;
+ float scale = ldexpf(1.0f, e);
+ for (size_t i = 0; i < count; i++)
+ out[base + i] = ds_e4m3_decode(ds_e4m3_encode(fmaxf(-448.0f, fminf(448.0f, in[base + i] / scale)))) * scale;
+ }
+}
+static float ds_e2m1_decode(int n) {
+ static const float v[16] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, 0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f};
+ return v[n & 15];
+}
+static void ds_fp4_qdq(float *out, const float *in, size_t n, size_t block) {
+ for (size_t base = 0; base < n; base += block) {
+ size_t count = n - base < block ? n - base : block;
+ float maximum = 0.0f;
+ for (size_t i = 0; i < count; i++) maximum = fmaxf(maximum, fabsf(in[base + i]));
+ maximum = fmaxf(maximum, 6.0f * ldexpf(1.0f, -126));
+ int e = ds_ceil_log2(maximum / 6.0f);
+ if (e < -127) e = -127;
+ if (e > 127) e = 127;
+ float scale = ldexpf(1.0f, e);
+ for (size_t i = 0; i < count; i++) {
+ float value = fmaxf(-6.0f, fminf(6.0f, in[base + i] / scale));
+ int best = 0; float distance = fabsf(value - ds_e2m1_decode(0));
+ for (int code = 1; code < 16; code++) {
+ float c = fabsf(value - ds_e2m1_decode(code));
+ if (c < distance) { distance = c; best = code; }
+ }
+ out[base + i] = ds_e2m1_decode(best) * scale;
+ }
+ }
+}
+static void ds_hadamard(float *v, size_t n) {
+ for (size_t w = 1; w < n; w *= 2)
+ for (size_t base = 0; base < n; base += 2 * w)
+ for (size_t i = 0; i < w; i++) { float l = v[base + i], r = v[base + w + i]; v[base + i] = l + r; v[base + w + i] = l - r; }
+ float scale = 1.0f / sqrtf((float)n);
+ for (size_t i = 0; i < n; i++) v[i] = bf16r(v[i] * scale);
+}
+static float ds_sigmoid(float v) { if (v >= 0.0f) { float d = expf(-v); return 1.0f / (1.0f + d); } float g = expf(v); return g / (1.0f + g); }
+
+/* A block's values: random magnitudes over 40 binades, and blocks built on purpose:
+ * all zero, the amax on the scale's boundary (1.75 * 2^E for E4M3, 1.5 * 2^E for E2M1)
+ * and one step above it, exact ties of the target grid (E4M3 mantissa midpoints and
+ * subnormal midpoints; E2M1 midpoints), signed zeros, an amax under E4M3's floor. (The
+ * bf16 rounding after E4M3 or E2M1 is the CPU's step but changes nothing: both grids
+ * are bf16 values.) */
+static void ds_fill(float *v, int n, int kind, int pattern) {
+ float s = ldexpf(1.0f, (int)(rnd() % 30) - 15);
+ for (int i = 0; i < n; i++) v[i] = frnd() * ldexpf(1.0f, (int)(rnd() % 40) - 20);
+ if (pattern == 1) for (int i = 0; i < n; i++) v[i] = 0.0f;
+ if (pattern == 2 || pattern == 3) {
+ float top = (kind == VKC_DS_E2M1 ? 1.5f : 1.75f) * 4.0f * s;
+ if (pattern == 3) { uint32_t u; memcpy(&u, &top, 4); u++; memcpy(&top, &u, 4); }
+ v[0] = top;
+ }
+ if (pattern == 4) {
+ float amax = (kind == VKC_DS_E2M1 ? 6.0f : 448.0f) * s;
+ static const float ties2[7] = {0.25f, 0.75f, 1.25f, 1.75f, 2.5f, 3.5f, 5.0f};
+ v[0] = amax;
+ for (int i = 1; i < n; i++) {
+ float t;
+ if (kind == VKC_DS_E2M1) t = ties2[rnd() % 7];
+ else if (rnd() % 3 == 0) t = (float)(2 * (int)(rnd() % 8) + 1) * ldexpf(1.0f, -10); /* subnormal midpoints */
+ else t = (1.0f + (float)(2 * (int)(rnd() % 8) + 1) / 16.0f) * ldexpf(1.0f, (int)(rnd() % 14) - 6);
+ v[i] = (rnd() & 1 ? -t : t) * s;
+ }
+ v[n / 2] = -0.0f; if (n > 3) v[3] = 0.0f;
+ }
+ if (pattern == 5) for (int i = 0; i < n; i++) v[i] = frnd() * ldexpf(1.0f, -20 - (int)(rnd() % 4)); /* under the floor */
+}
+/* kind, nseg segments of len (per_row a row, at strides), block, flags; in place or not */
+static void test_ds_round(int kind, int nseg, int per_row, int len, int block, int flags, int inplace) {
+ int rows = (nseg + per_row - 1) / per_row, xseg = len + 3, xrow = per_row * xseg + 5, yseg = len + 1, yrow = per_row * yseg + 2;
+ size_t xn = (size_t)rows * xrow + 7, yn = inplace ? xn : (size_t)rows * yrow + 4;
+ int xo = 7, yo = inplace ? 7 : 4, ys = inplace ? xseg : yseg, yr = inplace ? xrow : yrow;
+ float *x = fvec(xn, 1.f), *ref = malloc(xn > yn ? xn * 4 : yn * 4), *seg = malloc((size_t)len * 4);
+ int bl = kind == VKC_DS_E4M3 || kind == VKC_DS_E2M1 ? block : len, nb = (len + bl - 1) / bl;
+ for (int g = 0; g < nseg; g++) {
+ float *xs = x + xo + (g / per_row) * xrow + (g % per_row) * xseg;
+ for (int b = 0; b < nb; b++) ds_fill(xs + b * bl, len - b * bl < bl ? len - b * bl : bl, kind, (g * nb + b) % 6);
+ }
+ if (inplace) memcpy(ref, x, xn * 4);
+ else for (size_t i = 0; i < yn; i++) ref[i] = -7.0f; /* what the op must not touch */
+ for (int g = 0; g < nseg; g++) {
+ const float *xs = x + xo + (g / per_row) * xrow + (g % per_row) * xseg;
+ float *ys2 = ref + yo + (g / per_row) * yr + (g % per_row) * ys;
+ memcpy(seg, xs, (size_t)len * 4);
+ if (kind == VKC_DS_BF16) for (int i = 0; i < len; i++) seg[i] = bf16r(seg[i]);
+ else if (kind == VKC_DS_E4M3) ds_fp8_qdq(seg, xs, (size_t)len, (size_t)block);
+ else if (kind == VKC_DS_E2M1) ds_fp4_qdq(seg, xs, (size_t)len, (size_t)block);
+ else ds_hadamard(seg, (size_t)len);
+ if ((flags & 1) && (kind == VKC_DS_E4M3 || kind == VKC_DS_E2M1)) for (int i = 0; i < len; i++) seg[i] = bf16r(seg[i]);
+ memcpy(ys2, seg, (size_t)len * 4);
+ }
+ VkcBuf *xb = up(x, xn), *yb = inplace ? xb : NULL;
+ if (!inplace) { float *init = malloc(yn * 4); for (size_t i = 0; i < yn; i++) init[i] = -7.0f; yb = up(init, yn); free(init); }
+ VkcDsRound p = {kind, nseg, per_row, len, block, flags, xo, xrow, xseg, yo, yr, ys, 1.0f / sqrtf((float)len)};
+ vkc_begin(); int ok = vkc_dsv4_round(xb, yb, &p); vkc_submit(1);
+ float *got = down(yb, 0, yn);
+ size_t diff = 0, first = 0;
+ for (size_t i = 0; i < yn; i++) if (memcmp(&got[i], &ref[i], 4)) { if (!diff) first = i; diff++; }
+ CHECK(ok && !diff, "dsv4 round kind %d nseg %d len %d block %d flags %d inplace %d: %zu of %zu values differ (first at %zu: %a, want %a)",
+ kind, nseg, len, block, flags, inplace, diff, yn, first, (double)got[first], (double)ref[first]);
+ vkc_free(xb); if (!inplace) vkc_free(yb);
+ free(x); free(ref); free(seg); free(got);
+}
+/* The SwiGLU: within one bf16 step of the CPU (the device's exp is not the CPU's expf),
+ * and how many values are the CPU's bits */
+static void test_ds_swiglu(int n, float lim) {
+ float *a = fvec((size_t)n + 3, 14.f), *b = fvec((size_t)n + 5, 14.f), *ref = malloc((size_t)n * 4);
+ for (int i = 0; i < n; i++) {
+ float g = bf16r(a[3 + i]), u = bf16r(b[5 + i]);
+ if (lim > 0.0f) { g = fminf(g, lim); u = fmaxf(-lim, fminf(u, lim)); }
+ ref[i] = bf16r(g * ds_sigmoid(g) * u);
+ }
+ VkcBuf *ab = up(a, (size_t)n + 3), *bb = up(b, (size_t)n + 5), *yb = vkc_buf((size_t)(n + 2) * 4, VKC_DEV);
+ VkcDsSwiglu p = {n, 3, 5, 2, lim};
+ vkc_begin(); int ok = vkc_dsv4_swiglu(ab, bb, yb, &p); vkc_submit(1);
+ float *got = down(yb, 2, (size_t)n);
+ int same = 0, far = 0;
+ for (int i = 0; i < n; i++) {
+ uint32_t u, v; memcpy(&u, &got[i], 4); memcpy(&v, &ref[i], 4);
+ same += u == v;
+ long d = (long)(u >> 16) - (long)(v >> 16);
+ if ((u & 0xffff) || d > 1 || d < -1) far++;
+ }
+ CHECK(ok && !far, "dsv4 swiglu n %d lim %g: %d values further than one bf16 step (or not bf16)", n, lim, far);
+ printf(" dsv4 swiglu n %d lim %g: %d of %d values the CPU's bits, the rest one bf16 step away\n", n, lim, same, n);
+ vkc_free(ab); vkc_free(bb); vkc_free(yb);
+ free(a); free(b); free(ref); free(got);
+}
+
/* ---- frames: many ops, frames in flight, ordering ---------------------------------------- */
static void test_frames(void) {
int n = 1000;
@@ -1373,6 +1548,17 @@ int main(int argc, char **argv) {
test_ds_index(2, 900, 1, 3, 64, 100, 4, 8, 0); test_ds_index(4, 300, 4, 16, 64, 16, 1, 3, 1); test_ds_index(3, 1, 2, 2, 16, 4, 2, 1, 0);
test_ds_engram(1, 4, 128); test_ds_engram(3, 2, 300);
printf("dsv4 done\n");
+ int in;
+ for (in = 0; in < 2; in++) {
+ test_ds_round(VKC_DS_BF16, 6, 2, 50, 0, 0, in); test_ds_round(VKC_DS_BF16, 300, 1, 1000, 0, 0, in);
+ test_ds_round(VKC_DS_E4M3, 6, 2, 300, 128, 0, in); test_ds_round(VKC_DS_E4M3, 9, 3, 448, 64, 1, in);
+ test_ds_round(VKC_DS_E4M3, 40, 1, 4096, 128, 0, in); test_ds_round(VKC_DS_E4M3, 5, 1, 70, 256, 1, in);
+ test_ds_round(VKC_DS_E2M1, 6, 2, 128, 32, 1, in); test_ds_round(VKC_DS_E2M1, 10, 5, 100, 32, 0, in);
+ test_ds_round(VKC_DS_HADAMARD, 6, 2, 128, 0, 0, in); test_ds_round(VKC_DS_HADAMARD, 3, 1, 4096, 0, 0, in);
+ test_ds_round(VKC_DS_HADAMARD, 4, 4, 32, 0, 0, in); test_ds_round(VKC_DS_HADAMARD, 2, 1, 1, 0, 0, in);
+ }
+ test_ds_swiglu(5000, 10.0f); test_ds_swiglu(3000, 0.0f);
+ printf("dsv4 rounding done\n");
}
VkcStats st; vkc_stats(&st);
printf("chain: %llu frames, %llu ops, %llu matmuls (%llu GEMM), %llu barriers\n", st.frames, st.ops, st.matmuls, st.gemms, st.barriers);
diff --git a/c/vk_chain.c b/c/vk_chain.c
index 2d487f69..1c802022 100644
--- a/c/vk_chain.c
+++ b/c/vk_chain.c
@@ -1115,3 +1115,28 @@ int vkc_dsv4_engram(VkcBuf *kv, VkcBuf *prm, VkcBuf *x, const VkcDsEngram *p) {
VkcBind bd[6] = {B(kv, 0), B(prm, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(x, 1)};
return dsv4_rec(PK_EW, 6, p, sizeof *p, bd, 6, (uint32_t)p->H, (uint32_t)p->S);
}
+int vkc_dsv4_round(VkcBuf *x, VkcBuf *y, const VkcDsRound *p) {
+ if (!x || !y || p->kind < VKC_DS_BF16 || p->kind > VKC_DS_HADAMARD || p->nseg < 0 || p->per_row < 1 || p->len < 1)
+ return 0;
+ if ((p->kind == VKC_DS_E4M3 || p->kind == VKC_DS_E2M1) && (p->block < 1 || p->block > 256)) return 0;
+ if (p->kind == VKC_DS_HADAMARD && (p->len > 4096 || (p->len & (p->len - 1)))) return 0;
+ if (p->nseg == 0) return open_frame() && !K.lost;
+ /* the fields, then whether it runs in place (x is read where y is written), then hscale */
+ int w[14];
+ memcpy(w, p, 12 * sizeof(int));
+ w[12] = x == y;
+ memcpy(&w[13], &p->hscale, sizeof(float));
+ uint64_t groups = p->kind == VKC_DS_BF16 ? ((uint64_t)p->nseg * p->len + 255) / 256
+ : p->kind == VKC_DS_HADAMARD ? (uint64_t)p->nseg
+ : (uint64_t)p->nseg * ((p->len + p->block - 1) / p->block);
+ uint32_t gx, gy; grid(groups, &gx, &gy);
+ VkcBind bd[6] = {B(x == y ? NULL : x, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(y, 1)};
+ return dsv4_rec(PK_EW, 7, w, sizeof w, bd, 6, gx, gy);
+}
+int vkc_dsv4_swiglu(VkcBuf *a, VkcBuf *b, VkcBuf *y, const VkcDsSwiglu *p) {
+ if (!a || !b || !y || p->n < 0) return 0;
+ if (p->n == 0) return open_frame() && !K.lost;
+ uint32_t gx, gy; grid(((uint64_t)p->n + 255) / 256, &gx, &gy);
+ VkcBind bd[6] = {B(a, 0), B(b, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(y, 1)};
+ return dsv4_rec(PK_EW, 8, p, sizeof *p, bd, 6, gx, gy);
+}
diff --git a/c/vk_chain.h b/c/vk_chain.h
index c21c9104..14d81e18 100644
--- a/c/vk_chain.h
+++ b/c/vk_chain.h
@@ -341,6 +341,29 @@ typedef struct { int S, width, topk, sc_row, l_off, l_row, base, order; } VkcDsT
int vkc_dsv4_topk(VkcBuf *sc, VkcBuf *list, const VkcDsTopk *p);
typedef struct { int S, H, D, kv_off, kv_row, x_off, x_row, qw_off, kw_off; float eps; } VkcDsEngram;
int vkc_dsv4_engram(VkcBuf *kv, VkcBuf *prm, VkcBuf *x, const VkcDsEngram *p);
+/* DeepSeek V4's roundings, each the engine's C bit for bit (chain_dsv4.comp modes 7, 8):
+ * vkc_dsv4_round nseg segments of len floats (segment g at row g / per_row, index
+ * g % per_row: x_off + row*x_row + j*x_seg, y likewise) from x into y
+ * (y may be x: in place), as `kind`:
+ * VKC_DS_BF16 to bf16, nearest even (coli_bf16_round)
+ * VKC_DS_E4M3 E4M3 per block of `block` values with one
+ * power-of-two scale (coli_fp8_activation_qdq_ref)
+ * VKC_DS_E2M1 E2M1 per block (coli_fp4_activation_qdq_ref)
+ * VKC_DS_HADAMARD the Hadamard transform times hscale (the host's
+ * 1 / sqrtf(len)), to bf16 (coli_hadamard_bf16_ref)
+ * flags 1: the E4M3 or E2M1 result to bf16 after. Limits: block <= 256;
+ * the Hadamard's len a power of two up to 4096.
+ * vkc_dsv4_swiglu y[y_off + i] = bf16(g * sigmoid(g) * u), g = bf16(a[a_off + i]) and
+ * u = bf16(b[b_off + i]) clamped by lim when lim > 0, i < n (the shared
+ * expert's activation between its roundings; coli_v4_swiglu). */
+#define VKC_DS_BF16 0
+#define VKC_DS_E4M3 1
+#define VKC_DS_E2M1 2
+#define VKC_DS_HADAMARD 3
+typedef struct { int kind, nseg, per_row, len, block, flags, x_off, x_row, x_seg, y_off, y_row, y_seg; float hscale; } VkcDsRound;
+int vkc_dsv4_round(VkcBuf *x, VkcBuf *y, const VkcDsRound *p);
+typedef struct { int n, a_off, b_off, y_off; float lim; } VkcDsSwiglu;
+int vkc_dsv4_swiglu(VkcBuf *a, VkcBuf *b, VkcBuf *y, const VkcDsSwiglu *p);
/* counters, for the engines' [VK] lines */
typedef struct {