Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: DeepSeek V4's roundings on the chain
JustVugg32185798b4e4
diff --git a/c/shaders/chain_dsv4.comp b/c/shaders/chain_dsv4.compindex 1e210b19..f6d6c04a 100644--- a/c/shaders/chain_dsv4.comp+++ b/c/shaders/chain_dsv4.comp@@ -1,7 +1,7 @@#version 450// DeepSeek V4.1 Flash's (and DeepSeek V4's) attention on the dense chain (vk_chain.c,// chain_dsv4; vk_chain.h: vkc_dsv4_*). The model is MQA over one KV row per position:-// a sliding window of raw rows plus compressed rows a DSA indexer picks. Seven modes:+// a sliding window of raw rows plus compressed rows a DSA indexer picks. Nine modes://// mode 0, sparse attention with a sink (sparse_attn.h, coli_sparse_attend): one// workgroup per (head h, row s). The row's list is list[l_off + s*l_row + j], j < cnt:@@ -46,6 +46,27 @@// s): x = the stream (x_off + s*x_row + c*D), key = kv[s][c*D..], value = kv[s][H*D..];// r = 1/sqrt(mean(x^2) + eps) * 1/sqrt(mean(key^2) + eps), t = sum(x * qw * kw * key)// * r / sqrt(D), g = sigmoid(sign(t) * sqrt(max(|t|, 1e-6))); x += g * value.+// mode 7, DeepSeek V4's roundings over segments: segment g (row g / per_row, index+// j = g % per_row) is len floats at x_off + row*x_row + j*x_seg, written to y at+// y_off + row*y_row + j*y_seg (in place when x is y). Each kind as the engine's C:+// kind 0, bf16 (coli_bf16_round). One thread per value.+// kind 1, E4M3 per block of `block` values (coli_fp8_activation_qdq_ref): amax floored+// at 1e-4, the scale 2^e with e the smallest for which amax <= 448 * 2^e, each value+// times 2^-e clamped to +-448, rounded to E4M3 (nearest even, steps of 2^-9 below+// 2^-6) and times 2^e back. One workgroup per (segment, block), block <= 256.+// kind 2, E2M1 per block (coli_fp4_activation_qdq_ref): amax floored at 6 * 2^-126, the+// scale 2^e smallest for which amax <= 6 * 2^e, each value times 2^-e clamped to +-6+// and rounded to the nearest of 0, 0.5, 1, 1.5, 2, 3, 4, 6 (a tie to the smaller,+// zero as +0), times 2^e. One workgroup per (segment, block), block <= 256.+// kind 3, the Hadamard transform (coli_hadamard_bf16_ref): butterflies of width 1, 2,+// 4, ... in place in the CPU's order, then each value times hscale (the host's+// 1 / sqrtf(len)) to bf16. One workgroup per segment, len a power of two <= 4096.+// flags 1 (kinds 1 and 2): the result to bf16 after. Every step is exact in f32 or+// rounds once as the CPU does, so the result is the CPU's bit for bit.+// mode 8, DeepSeek V4's SwiGLU between the shared expert's roundings (coli_v4_swiglu):+// over n values g = bf16(a[a_off + i]), u = bf16(b[b_off + i]); with lim > 0,+// g = min(g, lim) and u = clamp(u, -lim, lim); y[y_off + i] = bf16(g * sigmoid(g) * u),+// the sigmoid in the CPU's stable form.// Offsets and strides in floats (ints for lists and masks).#extension GL_KHR_shader_subgroup_basic : enablelayout(local_size_x = 256) in;@@ -394,6 +415,108 @@ void engram() {for (int i = tid; i < D; i += 256) y5[xb + i] = y5[xb + i] + g * b0[vb + i];}+// ---- mode 7 ------------------------------------------------------------------------+float pow2(int k) { return uintBitsToFloat(uint(k + 127) << 23); } // 2^k, -126 <= k <= 127+// |x| <= 448: the nearest E4M3 value (ties to even), coli_e4m3fn_encode then _decode+float e4m3r(float x) {+ float a = abs(x), r;+ if (a == 0.0) return x;+ if (a >= 448.0) r = 448.0;+ else if (a < 0.015625) { // below 2^-6: subnormal codes, steps of 2^-9+ float s = a * 512.0, f = floor(s), fr = s - f;+ if (fr > 0.5 || (fr == 0.5 && (int(f) & 1) == 1)) f += 1.0;+ r = f * 0.001953125;+ } else { // 1.mmm x 2^E, three mantissa bits+ uint b = floatBitsToUint(a);+ int E = int((b >> 23) & 0xffu) - 127;+ uint sg = 0x800000u | (b & 0x7fffffu), q = sg >> 20, rem = sg & 0xfffffu;+ if (rem > 0x80000u || (rem == 0x80000u && (q & 1u) == 1u)) q++;+ r = float(q) * pow2(E - 3);+ }+ return x < 0.0 ? -r : r;+}+// |x| <= 6: the nearest E2M1 value, a tie to the smaller magnitude, zero as +0 (the CPU+// scans the codes in order and keeps the first of two equal distances)+float e2m1r(float x) {+ float a = abs(x), r;+ if (a <= 0.25) return 0.0;+ if (a <= 0.75) r = 0.5;+ else if (a <= 1.25) r = 1.0;+ else if (a <= 1.75) r = 1.5;+ else if (a <= 2.5) r = 2.0;+ else if (a <= 3.5) r = 3.0;+ else if (a <= 5.0) r = 4.0;+ else r = 6.0;+ return x < 0.0 ? -r : r;+}+float rd_in(int i) { return p.v[13] != 0 ? y5[i] : b0[i]; }+void roundings() {+ int g = int(gl_WorkGroupID.x + gl_WorkGroupID.y * gl_NumWorkGroups.x), tid = int(gl_LocalInvocationID.x);+ int kind = p.v[1], nseg = p.v[2], per_row = p.v[3], len = p.v[4], blk = p.v[5], flags = p.v[6];+ if (kind == 0) {+ int t = g * 256 + tid;+ if (t >= nseg * len) return;+ int sg = t / len, i = t - sg * len, row = sg / per_row, j = sg - row * per_row;+ float v = rd_in(p.v[7] + row * p.v[8] + j * p.v[9] + i);+ y5[p.v[10] + row * p.v[11] + j * p.v[12] + i] = bf16r(v);+ return;+ }+ if (kind == 3) { // one workgroup per segment+ if (g >= nseg) return;+ int row = g / per_row, j = g - row * per_row;+ int xb = p.v[7] + row * p.v[8] + j * p.v[9], yb = p.v[10] + row * p.v[11] + j * p.v[12];+ for (int i = tid; i < len; i += 256) sm[i] = rd_in(xb + i);+ barrier();+ for (int w = 1; w < len; w *= 2) {+ for (int k = tid; k < len / 2; k += 256) {+ int base = (k / w) * 2 * w, i = k - (k / w) * w;+ float l = sm[base + i], r = sm[base + w + i];+ sm[base + i] = l + r;+ sm[base + w + i] = l - r;+ }+ barrier();+ }+ float hs = intBitsToFloat(p.v[14]);+ for (int i = tid; i < len; i += 256) y5[yb + i] = bf16r(sm[i] * hs);+ return;+ }+ // kinds 1 and 2: one workgroup per (segment, block)+ int nblk = (len + blk - 1) / blk, sg = g / nblk, bk = g - sg * nblk;+ if (sg >= nseg) return;+ int row = sg / per_row, j = sg - row * per_row, i = bk * blk + tid;+ bool live = tid < blk && i < len;+ int xa = p.v[7] + row * p.v[8] + j * p.v[9] + i, ya = p.v[10] + row * p.v[11] + j * p.v[12] + i;+ float v = live ? rd_in(xa) : 0.0;+ red[tid] = abs(v);+ barrier();+ for (int off = 128; off > 0; off >>= 1) {+ if (tid < off) red[tid] = max(red[tid], red[tid + off]);+ barrier();+ }+ // m = (1 + man / 2^23) * 2^E. E4M3: 448 = 1.75 * 2^8, so m <= 448 * 2^e holds from+ // e = E - 8 when the mantissa is at most 1.75, else from E - 7; E2M1: 6 = 1.5 * 2^2,+ // from E - 2 when it is at most 1.5, else from E - 1. (The CPU takes the ceiling of+ // log2 of m / 448 or m / 6, a correctly rounded quotient: the two agree.)+ float m = kind == 1 ? max(red[0], 1.0e-4) : max(red[0], 6.0 * pow2(-126));+ uint mb = floatBitsToUint(m), man = mb & 0x7fffffu;+ int E = int((mb >> 23) & 0xffu) - 127;+ int e = kind == 1 ? (man <= 0x600000u ? E - 8 : E - 7) : (man <= 0x400000u ? E - 2 : E - 1);+ e = clamp(e, -126, 126);+ float r = kind == 1 ? e4m3r(clamp(v * pow2(-e), -448.0, 448.0)) * pow2(e)+ : e2m1r(clamp(v * pow2(-e), -6.0, 6.0)) * pow2(e);+ if ((flags & 1) != 0) r = bf16r(r);+ if (live) y5[ya] = r;+}++// ---- mode 8 ------------------------------------------------------------------------+void swiglu_v4() {+ int t = int(gl_GlobalInvocationID.x + gl_GlobalInvocationID.y * gl_NumWorkGroups.x * 256u);+ if (t >= p.v[1]) return;+ float g = bf16r(b0[p.v[2] + t]), u = bf16r(b1[p.v[3] + t]), lim = intBitsToFloat(p.v[5]);+ if (lim > 0.0) { g = min(g, lim); u = max(-lim, min(u, lim)); }+ y5[p.v[4] + t] = bf16r(g * sig(g) * u);+}+void main() {int m = MODE;if (m == 0) attend();@@ -403,4 +526,6 @@ void main() {else if (m == 4) candidates();else if (m == 5) topk();else if (m == 6) engram();+ else if (m == 7) roundings();+ else if (m == 8) swiglu_v4();}diff --git a/c/shaders/chain_dsv4.spv b/c/shaders/chain_dsv4.spvindex cd2546b6..b9386e22 100644Binary files a/c/shaders/chain_dsv4.spv and b/c/shaders/chain_dsv4.spv differdiff --git a/c/tests/test_vk_chain.c b/c/tests/test_vk_chain.cindex 2b4a98bc..495730d9 100644--- a/c/tests/test_vk_chain.c+++ b/c/tests/test_vk_chain.c@@ -36,6 +36,8 @@* window and compressed rows (V4's bf16 roundings too), interleaved RoPE* both ways, the compressor's ring (V4's overlapping form), the indexer's* scores, candidate blocks and top-k slot for slot, the engram gate+ * dsv4 rounding DeepSeek V4's roundings (deepseek_v4.c): bf16, E4M3 and E2M1 per block,+ * the Hadamard transform, bit for bit; its SwiGLU within one bf16 step** make vk-chain-check VK=1 (VK_ICD_FILENAMES=.../lvp_icd.json for Lavapipe) */#include <stdio.h>@@ -1245,6 +1247,179 @@ static void test_ds_engram(int S, int H, int D) {free(kv); free(x); free(prm); free(ref); free(got);}+/* ---- DeepSeek V4's roundings (chain_dsv4.comp modes 7, 8) ------------------------------+ * References transcribed from deepseek_v4.c (COLI_V4_UNIT_NATIVE_QUANT, the MATH unit's+ * coli_v4_swiglu): the device must give their bits. */+static float ds_e4m3_decode(uint8_t v) {+ int sign = v >> 7, exponent = (v >> 3) & 15, mantissa = v & 7;+ if (exponent == 15 && mantissa == 7) return NAN;+ float n = !exponent ? ldexpf((float)mantissa, -9) : ldexpf(1.0f + (float)mantissa / 8.0f, exponent - 7);+ return sign ? -n : n;+}+static uint8_t ds_e4m3_encode(float value) {+ if (isnan(value)) return 0x7f;+ int negative = signbit(value) != 0;+ float magnitude = fabsf(value);+ if (!magnitude) return negative ? 0x80 : 0;+ if (magnitude >= 448.0f) return (uint8_t)((negative ? 0x80 : 0) | 0x7e);+ uint8_t best;+ if (magnitude < 0.015625f) {+ float scaled = magnitude * 512.0f;+ uint8_t rounded = (uint8_t)scaled;+ float fraction = scaled - rounded;+ if (fraction > 0.5f || (fraction == 0.5f && (rounded & 1))) rounded++;+ best = rounded;+ } else {+ uint32_t bits; memcpy(&bits, &magnitude, 4);+ int exponent = (int)((bits >> 23) & 0xff) - 127;+ uint32_t significand = 0x800000u | (bits & 0x7fffffu), rounded = significand >> 20, remainder = significand & 0xfffffu;+ if (remainder > 0x80000u || (remainder == 0x80000u && (rounded & 1u))) rounded++;+ if (rounded == 16u) { rounded = 8u; exponent++; }+ best = (uint8_t)((exponent + 7) * 8 + (int)rounded - 8);+ }+ return (uint8_t)(best | (negative ? 0x80 : 0));+}+static int ds_ceil_log2(float value) { int e; float f = frexpf(value, &e); return f == 0.5f ? e - 1 : e; }+static void ds_fp8_qdq(float *out, const float *in, size_t n, size_t block) {+ for (size_t base = 0; base < n; base += block) {+ size_t count = n - base < block ? n - base : block;+ float maximum = 0.0f;+ for (size_t i = 0; i < count; i++) maximum = fmaxf(maximum, fabsf(in[base + i]));+ maximum = fmaxf(maximum, 1e-4f);+ int e = ds_ceil_log2(maximum / 448.0f);+ if (e < -127) e = -127;+ if (e > 127) e = 127;+ float scale = ldexpf(1.0f, e);+ for (size_t i = 0; i < count; i++)+ out[base + i] = ds_e4m3_decode(ds_e4m3_encode(fmaxf(-448.0f, fminf(448.0f, in[base + i] / scale)))) * scale;+ }+}+static float ds_e2m1_decode(int n) {+ static const float v[16] = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, 0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f};+ return v[n & 15];+}+static void ds_fp4_qdq(float *out, const float *in, size_t n, size_t block) {+ for (size_t base = 0; base < n; base += block) {+ size_t count = n - base < block ? n - base : block;+ float maximum = 0.0f;+ for (size_t i = 0; i < count; i++) maximum = fmaxf(maximum, fabsf(in[base + i]));+ maximum = fmaxf(maximum, 6.0f * ldexpf(1.0f, -126));+ int e = ds_ceil_log2(maximum / 6.0f);+ if (e < -127) e = -127;+ if (e > 127) e = 127;+ float scale = ldexpf(1.0f, e);+ for (size_t i = 0; i < count; i++) {+ float value = fmaxf(-6.0f, fminf(6.0f, in[base + i] / scale));+ int best = 0; float distance = fabsf(value - ds_e2m1_decode(0));+ for (int code = 1; code < 16; code++) {+ float c = fabsf(value - ds_e2m1_decode(code));+ if (c < distance) { distance = c; best = code; }+ }+ out[base + i] = ds_e2m1_decode(best) * scale;+ }+ }+}+static void ds_hadamard(float *v, size_t n) {+ for (size_t w = 1; w < n; w *= 2)+ for (size_t base = 0; base < n; base += 2 * w)+ for (size_t i = 0; i < w; i++) { float l = v[base + i], r = v[base + w + i]; v[base + i] = l + r; v[base + w + i] = l - r; }+ float scale = 1.0f / sqrtf((float)n);+ for (size_t i = 0; i < n; i++) v[i] = bf16r(v[i] * scale);+}+static float ds_sigmoid(float v) { if (v >= 0.0f) { float d = expf(-v); return 1.0f / (1.0f + d); } float g = expf(v); return g / (1.0f + g); }++/* A block's values: random magnitudes over 40 binades, and blocks built on purpose:+ * all zero, the amax on the scale's boundary (1.75 * 2^E for E4M3, 1.5 * 2^E for E2M1)+ * and one step above it, exact ties of the target grid (E4M3 mantissa midpoints and+ * subnormal midpoints; E2M1 midpoints), signed zeros, an amax under E4M3's floor. (The+ * bf16 rounding after E4M3 or E2M1 is the CPU's step but changes nothing: both grids+ * are bf16 values.) */+static void ds_fill(float *v, int n, int kind, int pattern) {+ float s = ldexpf(1.0f, (int)(rnd() % 30) - 15);+ for (int i = 0; i < n; i++) v[i] = frnd() * ldexpf(1.0f, (int)(rnd() % 40) - 20);+ if (pattern == 1) for (int i = 0; i < n; i++) v[i] = 0.0f;+ if (pattern == 2 || pattern == 3) {+ float top = (kind == VKC_DS_E2M1 ? 1.5f : 1.75f) * 4.0f * s;+ if (pattern == 3) { uint32_t u; memcpy(&u, &top, 4); u++; memcpy(&top, &u, 4); }+ v[0] = top;+ }+ if (pattern == 4) {+ float amax = (kind == VKC_DS_E2M1 ? 6.0f : 448.0f) * s;+ static const float ties2[7] = {0.25f, 0.75f, 1.25f, 1.75f, 2.5f, 3.5f, 5.0f};+ v[0] = amax;+ for (int i = 1; i < n; i++) {+ float t;+ if (kind == VKC_DS_E2M1) t = ties2[rnd() % 7];+ else if (rnd() % 3 == 0) t = (float)(2 * (int)(rnd() % 8) + 1) * ldexpf(1.0f, -10); /* subnormal midpoints */+ else t = (1.0f + (float)(2 * (int)(rnd() % 8) + 1) / 16.0f) * ldexpf(1.0f, (int)(rnd() % 14) - 6);+ v[i] = (rnd() & 1 ? -t : t) * s;+ }+ v[n / 2] = -0.0f; if (n > 3) v[3] = 0.0f;+ }+ if (pattern == 5) for (int i = 0; i < n; i++) v[i] = frnd() * ldexpf(1.0f, -20 - (int)(rnd() % 4)); /* under the floor */+}+/* kind, nseg segments of len (per_row a row, at strides), block, flags; in place or not */+static void test_ds_round(int kind, int nseg, int per_row, int len, int block, int flags, int inplace) {+ int rows = (nseg + per_row - 1) / per_row, xseg = len + 3, xrow = per_row * xseg + 5, yseg = len + 1, yrow = per_row * yseg + 2;+ size_t xn = (size_t)rows * xrow + 7, yn = inplace ? xn : (size_t)rows * yrow + 4;+ int xo = 7, yo = inplace ? 7 : 4, ys = inplace ? xseg : yseg, yr = inplace ? xrow : yrow;+ float *x = fvec(xn, 1.f), *ref = malloc(xn > yn ? xn * 4 : yn * 4), *seg = malloc((size_t)len * 4);+ int bl = kind == VKC_DS_E4M3 || kind == VKC_DS_E2M1 ? block : len, nb = (len + bl - 1) / bl;+ for (int g = 0; g < nseg; g++) {+ float *xs = x + xo + (g / per_row) * xrow + (g % per_row) * xseg;+ for (int b = 0; b < nb; b++) ds_fill(xs + b * bl, len - b * bl < bl ? len - b * bl : bl, kind, (g * nb + b) % 6);+ }+ if (inplace) memcpy(ref, x, xn * 4);+ else for (size_t i = 0; i < yn; i++) ref[i] = -7.0f; /* what the op must not touch */+ for (int g = 0; g < nseg; g++) {+ const float *xs = x + xo + (g / per_row) * xrow + (g % per_row) * xseg;+ float *ys2 = ref + yo + (g / per_row) * yr + (g % per_row) * ys;+ memcpy(seg, xs, (size_t)len * 4);+ if (kind == VKC_DS_BF16) for (int i = 0; i < len; i++) seg[i] = bf16r(seg[i]);+ else if (kind == VKC_DS_E4M3) ds_fp8_qdq(seg, xs, (size_t)len, (size_t)block);+ else if (kind == VKC_DS_E2M1) ds_fp4_qdq(seg, xs, (size_t)len, (size_t)block);+ else ds_hadamard(seg, (size_t)len);+ if ((flags & 1) && (kind == VKC_DS_E4M3 || kind == VKC_DS_E2M1)) for (int i = 0; i < len; i++) seg[i] = bf16r(seg[i]);+ memcpy(ys2, seg, (size_t)len * 4);+ }+ VkcBuf *xb = up(x, xn), *yb = inplace ? xb : NULL;+ if (!inplace) { float *init = malloc(yn * 4); for (size_t i = 0; i < yn; i++) init[i] = -7.0f; yb = up(init, yn); free(init); }+ VkcDsRound p = {kind, nseg, per_row, len, block, flags, xo, xrow, xseg, yo, yr, ys, 1.0f / sqrtf((float)len)};+ vkc_begin(); int ok = vkc_dsv4_round(xb, yb, &p); vkc_submit(1);+ float *got = down(yb, 0, yn);+ size_t diff = 0, first = 0;+ for (size_t i = 0; i < yn; i++) if (memcmp(&got[i], &ref[i], 4)) { if (!diff) first = i; diff++; }+ CHECK(ok && !diff, "dsv4 round kind %d nseg %d len %d block %d flags %d inplace %d: %zu of %zu values differ (first at %zu: %a, want %a)",+ kind, nseg, len, block, flags, inplace, diff, yn, first, (double)got[first], (double)ref[first]);+ vkc_free(xb); if (!inplace) vkc_free(yb);+ free(x); free(ref); free(seg); free(got);+}+/* The SwiGLU: within one bf16 step of the CPU (the device's exp is not the CPU's expf),+ * and how many values are the CPU's bits */+static void test_ds_swiglu(int n, float lim) {+ float *a = fvec((size_t)n + 3, 14.f), *b = fvec((size_t)n + 5, 14.f), *ref = malloc((size_t)n * 4);+ for (int i = 0; i < n; i++) {+ float g = bf16r(a[3 + i]), u = bf16r(b[5 + i]);+ if (lim > 0.0f) { g = fminf(g, lim); u = fmaxf(-lim, fminf(u, lim)); }+ ref[i] = bf16r(g * ds_sigmoid(g) * u);+ }+ VkcBuf *ab = up(a, (size_t)n + 3), *bb = up(b, (size_t)n + 5), *yb = vkc_buf((size_t)(n + 2) * 4, VKC_DEV);+ VkcDsSwiglu p = {n, 3, 5, 2, lim};+ vkc_begin(); int ok = vkc_dsv4_swiglu(ab, bb, yb, &p); vkc_submit(1);+ float *got = down(yb, 2, (size_t)n);+ int same = 0, far = 0;+ for (int i = 0; i < n; i++) {+ uint32_t u, v; memcpy(&u, &got[i], 4); memcpy(&v, &ref[i], 4);+ same += u == v;+ long d = (long)(u >> 16) - (long)(v >> 16);+ if ((u & 0xffff) || d > 1 || d < -1) far++;+ }+ CHECK(ok && !far, "dsv4 swiglu n %d lim %g: %d values further than one bf16 step (or not bf16)", n, lim, far);+ printf(" dsv4 swiglu n %d lim %g: %d of %d values the CPU's bits, the rest one bf16 step away\n", n, lim, same, n);+ vkc_free(ab); vkc_free(bb); vkc_free(yb);+ free(a); free(b); free(ref); free(got);+}+/* ---- frames: many ops, frames in flight, ordering ---------------------------------------- */static void test_frames(void) {int n = 1000;@@ -1373,6 +1548,17 @@ int main(int argc, char **argv) {test_ds_index(2, 900, 1, 3, 64, 100, 4, 8, 0); test_ds_index(4, 300, 4, 16, 64, 16, 1, 3, 1); test_ds_index(3, 1, 2, 2, 16, 4, 2, 1, 0);test_ds_engram(1, 4, 128); test_ds_engram(3, 2, 300);printf("dsv4 done\n");+ int in;+ for (in = 0; in < 2; in++) {+ test_ds_round(VKC_DS_BF16, 6, 2, 50, 0, 0, in); test_ds_round(VKC_DS_BF16, 300, 1, 1000, 0, 0, in);+ test_ds_round(VKC_DS_E4M3, 6, 2, 300, 128, 0, in); test_ds_round(VKC_DS_E4M3, 9, 3, 448, 64, 1, in);+ test_ds_round(VKC_DS_E4M3, 40, 1, 4096, 128, 0, in); test_ds_round(VKC_DS_E4M3, 5, 1, 70, 256, 1, in);+ test_ds_round(VKC_DS_E2M1, 6, 2, 128, 32, 1, in); test_ds_round(VKC_DS_E2M1, 10, 5, 100, 32, 0, in);+ test_ds_round(VKC_DS_HADAMARD, 6, 2, 128, 0, 0, in); test_ds_round(VKC_DS_HADAMARD, 3, 1, 4096, 0, 0, in);+ test_ds_round(VKC_DS_HADAMARD, 4, 4, 32, 0, 0, in); test_ds_round(VKC_DS_HADAMARD, 2, 1, 1, 0, 0, in);+ }+ test_ds_swiglu(5000, 10.0f); test_ds_swiglu(3000, 0.0f);+ printf("dsv4 rounding done\n");}VkcStats st; vkc_stats(&st);printf("chain: %llu frames, %llu ops, %llu matmuls (%llu GEMM), %llu barriers\n", st.frames, st.ops, st.matmuls, st.gemms, st.barriers);diff --git a/c/vk_chain.c b/c/vk_chain.cindex 2d487f69..1c802022 100644--- a/c/vk_chain.c+++ b/c/vk_chain.c@@ -1115,3 +1115,28 @@ int vkc_dsv4_engram(VkcBuf *kv, VkcBuf *prm, VkcBuf *x, const VkcDsEngram *p) {VkcBind bd[6] = {B(kv, 0), B(prm, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(x, 1)};return dsv4_rec(PK_EW, 6, p, sizeof *p, bd, 6, (uint32_t)p->H, (uint32_t)p->S);}+int vkc_dsv4_round(VkcBuf *x, VkcBuf *y, const VkcDsRound *p) {+ if (!x || !y || p->kind < VKC_DS_BF16 || p->kind > VKC_DS_HADAMARD || p->nseg < 0 || p->per_row < 1 || p->len < 1)+ return 0;+ if ((p->kind == VKC_DS_E4M3 || p->kind == VKC_DS_E2M1) && (p->block < 1 || p->block > 256)) return 0;+ if (p->kind == VKC_DS_HADAMARD && (p->len > 4096 || (p->len & (p->len - 1)))) return 0;+ if (p->nseg == 0) return open_frame() && !K.lost;+ /* the fields, then whether it runs in place (x is read where y is written), then hscale */+ int w[14];+ memcpy(w, p, 12 * sizeof(int));+ w[12] = x == y;+ memcpy(&w[13], &p->hscale, sizeof(float));+ uint64_t groups = p->kind == VKC_DS_BF16 ? ((uint64_t)p->nseg * p->len + 255) / 256+ : p->kind == VKC_DS_HADAMARD ? (uint64_t)p->nseg+ : (uint64_t)p->nseg * ((p->len + p->block - 1) / p->block);+ uint32_t gx, gy; grid(groups, &gx, &gy);+ VkcBind bd[6] = {B(x == y ? NULL : x, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(y, 1)};+ return dsv4_rec(PK_EW, 7, w, sizeof w, bd, 6, gx, gy);+}+int vkc_dsv4_swiglu(VkcBuf *a, VkcBuf *b, VkcBuf *y, const VkcDsSwiglu *p) {+ if (!a || !b || !y || p->n < 0) return 0;+ if (p->n == 0) return open_frame() && !K.lost;+ uint32_t gx, gy; grid(((uint64_t)p->n + 255) / 256, &gx, &gy);+ VkcBind bd[6] = {B(a, 0), B(b, 0), B(NULL, 0), B(NULL, 0), B(NULL, 0), B(y, 1)};+ return dsv4_rec(PK_EW, 8, p, sizeof *p, bd, 6, gx, gy);+}diff --git a/c/vk_chain.h b/c/vk_chain.hindex c21c9104..14d81e18 100644--- a/c/vk_chain.h+++ b/c/vk_chain.h@@ -341,6 +341,29 @@ typedef struct { int S, width, topk, sc_row, l_off, l_row, base, order; } VkcDsTint vkc_dsv4_topk(VkcBuf *sc, VkcBuf *list, const VkcDsTopk *p);typedef struct { int S, H, D, kv_off, kv_row, x_off, x_row, qw_off, kw_off; float eps; } VkcDsEngram;int vkc_dsv4_engram(VkcBuf *kv, VkcBuf *prm, VkcBuf *x, const VkcDsEngram *p);+/* DeepSeek V4's roundings, each the engine's C bit for bit (chain_dsv4.comp modes 7, 8):+ * vkc_dsv4_round nseg segments of len floats (segment g at row g / per_row, index+ * g % per_row: x_off + row*x_row + j*x_seg, y likewise) from x into y+ * (y may be x: in place), as `kind`:+ * VKC_DS_BF16 to bf16, nearest even (coli_bf16_round)+ * VKC_DS_E4M3 E4M3 per block of `block` values with one+ * power-of-two scale (coli_fp8_activation_qdq_ref)+ * VKC_DS_E2M1 E2M1 per block (coli_fp4_activation_qdq_ref)+ * VKC_DS_HADAMARD the Hadamard transform times hscale (the host's+ * 1 / sqrtf(len)), to bf16 (coli_hadamard_bf16_ref)+ * flags 1: the E4M3 or E2M1 result to bf16 after. Limits: block <= 256;+ * the Hadamard's len a power of two up to 4096.+ * vkc_dsv4_swiglu y[y_off + i] = bf16(g * sigmoid(g) * u), g = bf16(a[a_off + i]) and+ * u = bf16(b[b_off + i]) clamped by lim when lim > 0, i < n (the shared+ * expert's activation between its roundings; coli_v4_swiglu). */+#define VKC_DS_BF16 0+#define VKC_DS_E4M3 1+#define VKC_DS_E2M1 2+#define VKC_DS_HADAMARD 3+typedef struct { int kind, nseg, per_row, len, block, flags, x_off, x_row, x_seg, y_off, y_row, y_seg; float hscale; } VkcDsRound;+int vkc_dsv4_round(VkcBuf *x, VkcBuf *y, const VkcDsRound *p);+typedef struct { int n, a_off, b_off, y_off; float lim; } VkcDsSwiglu;+int vkc_dsv4_swiglu(VkcBuf *a, VkcBuf *b, VkcBuf *y, const VkcDsSwiglu *p);/* counters, for the engines' [VK] lines */typedef struct {