Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 cudalab: DeepSeek kernel parity + speed harness (mHC, router, compressor, indexer, top-k, sparse attention, Engram) over src/dsv41/ds4_kernels.cu

Niko1221cb725bb20950
diff --git a/tools/dsv41_lab/build.sh b/tools/dsv41_lab/build.sh
index 7b7f1b980..5007bbf99 100644
--- a/tools/dsv41_lab/build.sh
+++ b/tools/dsv41_lab/build.sh
@@ -7,3 +7,5 @@ NVCC=${NVCC:-/usr/local/cuda-12.9/bin/nvcc}
mkdir -p out
g++ -O2 -std=c++17 -I../../third_party/ggml -c ../trellis/tqenc.cpp -o out/tqenc.o
$NVCC -O3 -std=c++17 -arch=$ARCH -lineinfo -o out/tqk_lab tqk_lab.cu out/tqenc.o -lpthread
+# DeepSeek kernel lab: links src/dsv41/ds4_kernels.cu unmodified
+$NVCC -O3 -std=c++17 -arch=$ARCH -lineinfo -I../../include -o out/ds4_lab ds4_lab.cu ../../src/dsv41/ds4_kernels.cu
diff --git a/tools/dsv41_lab/ds4_lab.cu b/tools/dsv41_lab/ds4_lab.cu
new file mode 100644
index 000000000..42f4e7eae
--- /dev/null
+++ b/tools/dsv41_lab/ds4_lab.cu
@@ -0,0 +1,372 @@
+// tools/dsv41_lab/ds4_lab.cu - parity + speed of the DeepSeek-V4.1 device kernels (src/dsv41/ds4_kernels.cu, the file the port
+// ports, linked unmodified) on synthetic tensors at the real shapes: dim 5120, hc 4 (mHC 24 x 20480, 20 Sinkhorn iterations),
+// 64 heads x 512 (MQA, K = V), 128-row window + sink, top-512 compressed rows (E2M1 + E4M3 scales), indexer 32 x 128,
+// router 384 / 6, Engram rows 256 (FP8 + E8M0). References are plain double-precision C++ written from the kernels' documented
+// math (DeepSeek model.py), not from the kernels.
+// ds4_lab [--tokens 1,2,3,4,5] [--only name,name] [--ctx 8192] [--reps 100]
+#include "../../src/dsv41/ds4_kernels.cuh"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <string>
+#include <vector>
+
+#define CK(x) do { cudaError_t e_ = (x); if (e_ != cudaSuccess) { std::printf("CUDA error %s at %s:%d\n", cudaGetErrorString(e_), __FILE__, __LINE__); std::exit(2); } } while (0)
+
+using namespace strata::ds4;
+
+static uint64_t g_rng = 0x2545f4914f6cdd1dull;
+static uint64_t rnd() { g_rng ^= g_rng << 13; g_rng ^= g_rng >> 7; g_rng ^= g_rng << 17; return g_rng; }
+static float urand() { return (float) ((rnd() >> 40) * (1.0 / 16777216.0)); } // [0, 1)
+static float srand_() { return urand() * 2.f - 1.f; } // [-1, 1)
+static float nrand() { float u = 0; for (int i = 0; i < 12; ++i) u += urand(); return u - 6.f; }
+
+static cudaStream_t g_s;
+static int g_fails = 0;
+static int g_reps = 100;
+
+template <class T> struct Dev {
+ T* p = nullptr; size_t n = 0;
+ Dev() {}
+ explicit Dev(const std::vector<T>& h) { set(h); }
+ void set(const std::vector<T>& h) { if (p) cudaFree(p); n = h.size(); CK(cudaMalloc(&p, std::max<size_t>(n, 1) * sizeof(T))); CK(cudaMemcpy(p, h.data(), n * sizeof(T), cudaMemcpyHostToDevice)); }
+ void alloc(size_t cnt) { if (p) cudaFree(p); n = cnt; CK(cudaMalloc(&p, std::max<size_t>(n, 1) * sizeof(T))); CK(cudaMemset(p, 0, n * sizeof(T))); }
+ std::vector<T> get() const { std::vector<T> h(n); CK(cudaMemcpy(h.data(), p, n * sizeof(T), cudaMemcpyDeviceToHost)); return h; }
+ ~Dev() { if (p) cudaFree(p); }
+};
+
+static uint16_t f2bf(float f) { uint32_t u; std::memcpy(&u, &f, 4); u += 0x7fffu + ((u >> 16) & 1u); return (uint16_t) (u >> 16); }
+static float bf2f(uint16_t b) { uint32_t u = (uint32_t) b << 16; float f; std::memcpy(&f, &u, 4); return f; }
+static float e4m3_val(uint8_t q) {
+ const int e = (q >> 3) & 15, m = q & 7; float v;
+ if (e == 15 && m == 7) v = NAN; else if (e == 0) v = m * (1.f / 512.f); else v = std::ldexp(1.f + m / 8.f, e - 7);
+ return (q & 0x80) ? -v : v;
+}
+static const float kE2m1[8] = {0.f, 0.5f, 1.f, 1.5f, 2.f, 3.f, 4.f, 6.f};
+static float e2m1_val(int c) { return (c & 8) ? -kE2m1[c & 7] : kE2m1[c & 7]; }
+
+// ---- timing: n launches of `fn` captured into a graph (no launch gaps), replayed
+static double time_us(const std::function<void()>& fn) {
+ cudaGraph_t g; cudaGraphExec_t ge;
+ const int inner = 20;
+ CK(cudaStreamBeginCapture(g_s, cudaStreamCaptureModeGlobal));
+ for (int i = 0; i < inner; ++i) fn();
+ CK(cudaStreamEndCapture(g_s, &g));
+ CK(cudaGraphInstantiate(&ge, g, 0));
+ CK(cudaGraphLaunch(ge, g_s)); CK(cudaStreamSynchronize(g_s));
+ cudaEvent_t e0, e1; cudaEventCreate(&e0); cudaEventCreate(&e1);
+ const int outer = std::max(1, g_reps / inner);
+ cudaEventRecord(e0, g_s);
+ for (int i = 0; i < outer; ++i) CK(cudaGraphLaunch(ge, g_s));
+ cudaEventRecord(e1, g_s); CK(cudaEventSynchronize(e1));
+ float ms; cudaEventElapsedTime(&ms, e0, e1);
+ cudaGraphExecDestroy(ge); cudaGraphDestroy(g);
+ return ms * 1e3 / (outer * inner);
+}
+
+static void report(const char* name, int T, double err, double tol, double us, const char* extra = "") {
+ const bool ok = err <= tol;
+ g_fails += !ok;
+ std::printf("%-22s T=%-2d err %.2e (tol %.0e) %-4s %9.2f us %s\n", name, T, err, tol, ok ? "ok" : "FAIL", us, extra);
+}
+static double max_rel(const std::vector<float>& got, const std::vector<double>& ref) { // max |got-ref| / (max|ref| + tiny)
+ double mx = 0, e = 0;
+ for (double r : ref) mx = std::max(mx, std::fabs(r));
+ for (size_t i = 0; i < ref.size(); ++i) e = std::max(e, std::fabs(got[i] - ref[i]));
+ return e / (mx + 1e-30);
+}
+
+// ============================================================================================================ mHC
+static void test_mhc(int T) {
+ const int dim = 5120, hc = 4, hcdim = hc * dim, n_mix = 24, iters = 20;
+ const float eps = 1e-6f, hc_eps = 1e-6f;
+ std::vector<float> h((size_t) T * hcdim), fn((size_t) n_mix * hcdim), scale = {0.3f, 0.5f, 0.7f}, base(24);
+ for (auto& v : h) v = nrand();
+ for (auto& v : fn) v = nrand() * 0.01f;
+ for (auto& v : base) v = srand_();
+ Dev<float> dh(h), dfn(fn), dscale(scale), dbase(base), dmix, dpre, dpost, dcomb;
+ dmix.alloc((size_t) T * 32); dpre.alloc((size_t) T * 8); dpost.alloc((size_t) T * 8); dcomb.alloc((size_t) T * 64);
+ // hc_mix
+ hc_mix(dh.p, dfn.p, hcdim, n_mix, T, eps, dmix.p, g_s); CK(cudaStreamSynchronize(g_s));
+ auto mix = dmix.get();
+ std::vector<double> rmix((size_t) T * 32, 0.0), gm((size_t) T * 32, 0.0);
+ for (int t = 0; t < T; ++t) {
+ double ss = 0; for (int k = 0; k < hcdim; ++k) ss += (double) h[(size_t) t * hcdim + k] * h[(size_t) t * hcdim + k];
+ const double inv = 1.0 / std::sqrt(ss / hcdim + eps);
+ for (int m = 0; m < n_mix; ++m) { double a = 0; for (int k = 0; k < hcdim; ++k) a += (double) h[(size_t) t * hcdim + k] * fn[(size_t) m * hcdim + k]; rmix[(size_t) t * 32 + m] = a * inv; gm[(size_t) t * 32 + m] = mix[(size_t) t * 32 + m]; }
+ }
+ std::vector<float> mixg(mix.begin(), mix.end());
+ { std::vector<double> r2; std::vector<float> g2; for (int t = 0; t < T; ++t) for (int m = 0; m < n_mix; ++m) { r2.push_back(rmix[(size_t) t * 32 + m]); g2.push_back(mix[(size_t) t * 32 + m]); }
+ report("hc_mix", T, max_rel(g2, r2), 1e-5, time_us([&] { hc_mix(dh.p, dfn.p, hcdim, n_mix, T, eps, dmix.p, g_s); }), "24 x 20480 dots + norm"); }
+ // hc_split (Sinkhorn)
+ hc_split(dmix.p, dscale.p, dbase.p, hc, iters, hc_eps, T, dpre.p, dpost.p, dcomb.p, g_s); CK(cudaStreamSynchronize(g_s));
+ auto pre = dpre.get(), post = dpost.get(), comb = dcomb.get();
+ {
+ std::vector<double> rr; std::vector<float> gg;
+ for (int t = 0; t < T; ++t) {
+ const float* mx = mix.data() + (size_t) t * 32;
+ for (int k = 0; k < hc; ++k) { rr.push_back(1.0 / (1.0 + std::exp(-((double) mx[k] * scale[0] + base[k]))) + hc_eps); gg.push_back(pre[(size_t) t * 8 + k]); }
+ for (int k = 0; k < hc; ++k) { rr.push_back(2.0 / (1.0 + std::exp(-((double) mx[hc + k] * scale[1] + base[hc + k])))); gg.push_back(post[(size_t) t * 8 + k]); }
+ double c[4][4];
+ for (int j = 0; j < 4; ++j) { double m = -1e300; for (int k = 0; k < 4; ++k) { c[j][k] = (double) mx[8 + j * 4 + k] * scale[2] + base[8 + j * 4 + k]; m = std::max(m, c[j][k]); }
+ double s = 0; for (int k = 0; k < 4; ++k) { c[j][k] = std::exp(c[j][k] - m); s += c[j][k]; } for (int k = 0; k < 4; ++k) c[j][k] = c[j][k] / s + hc_eps; }
+ auto colnorm = [&] { for (int k = 0; k < 4; ++k) { double s = 0; for (int j = 0; j < 4; ++j) s += c[j][k]; for (int j = 0; j < 4; ++j) c[j][k] /= s + hc_eps; } };
+ auto rownorm = [&] { for (int j = 0; j < 4; ++j) { double s = 0; for (int k = 0; k < 4; ++k) s += c[j][k]; for (int k = 0; k < 4; ++k) c[j][k] /= s + hc_eps; } };
+ colnorm();
+ for (int it = 1; it < iters; ++it) { rownorm(); colnorm(); }
+ for (int j = 0; j < 4; ++j) for (int k = 0; k < 4; ++k) { rr.push_back(c[j][k]); gg.push_back(comb[(size_t) t * 64 + j * 4 + k]); }
+ }
+ report("hc_split sinkhorn", T, max_rel(gg, rr), 2e-5, time_us([&] { hc_split(dmix.p, dscale.p, dbase.p, hc, iters, hc_eps, T, dpre.p, dpost.p, dcomb.p, g_s); }), "20 iterations");
+ }
+ // hc_pre_norm: x = rmsnorm(sum_s pre[s] h[s]) * w
+ std::vector<float> w(dim); for (auto& v : w) v = 0.5f + urand();
+ Dev<float> dw(w), dx; dx.alloc((size_t) T * dim);
+ hc_pre_norm(dh.p, dpre.p, dw.p, dim, hc, T, eps, dx.p, g_s); CK(cudaStreamSynchronize(g_s));
+ {
+ auto x = dx.get(); std::vector<double> rr; std::vector<float> gg;
+ for (int t = 0; t < T; ++t) {
+ std::vector<double> a(dim); double ss = 0;
+ for (int e = 0; e < dim; ++e) { double s = 0; for (int k = 0; k < hc; ++k) s += (double) pre[(size_t) t * 8 + k] * h[((size_t) t * hc + k) * dim + e]; a[e] = s; ss += s * s; }
+ const double inv = 1.0 / std::sqrt(ss / dim + eps);
+ for (int e = 0; e < dim; ++e) { rr.push_back(a[e] * inv * w[e]); gg.push_back(x[(size_t) t * dim + e]); }
+ }
+ report("hc_pre_norm", T, max_rel(gg, rr), 2e-5, time_us([&] { hc_pre_norm(dh.p, dpre.p, dw.p, dim, hc, T, eps, dx.p, g_s); }));
+ }
+ // hc_post / hc_post_inplace: out[k][e] = post[k] y[e] + sum_j comb[j][k] res[j][e]
+ std::vector<float> y((size_t) T * dim); for (auto& v : y) v = nrand();
+ Dev<float> dy(y), dout; dout.alloc((size_t) T * hcdim);
+ hc_post(dy.p, dh.p, dpost.p, dcomb.p, dim, hc, T, dout.p, g_s); CK(cudaStreamSynchronize(g_s));
+ {
+ auto o = dout.get(); std::vector<double> rr; std::vector<float> gg;
+ for (int t = 0; t < T; ++t) for (int k = 0; k < hc; ++k) for (int e = 0; e < dim; ++e) {
+ double a = (double) post[(size_t) t * 8 + k] * y[(size_t) t * dim + e];
+ for (int j = 0; j < hc; ++j) a += (double) comb[(size_t) t * 64 + j * hc + k] * h[((size_t) t * hc + j) * dim + e];
+ rr.push_back(a); gg.push_back(o[((size_t) t * hc + k) * dim + e]);
+ }
+ const double err = max_rel(gg, rr);
+ report("hc_post", T, err, 2e-6, time_us([&] { hc_post(dy.p, dh.p, dpost.p, dcomb.p, dim, hc, T, dout.p, g_s); }));
+ Dev<float> dh2(h);
+ hc_post_inplace(dy.p, dh2.p, dpost.p, dcomb.p, dim, hc, T, g_s); CK(cudaStreamSynchronize(g_s));
+ auto o2 = dh2.get(); std::vector<float> gg2(o2.begin(), o2.begin() + (size_t) T * hcdim);
+ report("hc_post_inplace", T, max_rel(gg2, rr), 2e-6, time_us([&] { hc_post_inplace(dy.p, dh2.p, dpost.p, dcomb.p, dim, hc, T, g_s); }), "(timing re-applies in place)");
+ }
+}
+
+// ============================================================================================================ router
+static void test_router(int T) {
+ const int n = 384, k = 6; const float rs = 1.5f;
+ std::vector<float> logits((size_t) T * 1024, 0.f), bias(n);
+ for (int t = 0; t < T; ++t) for (int i = 0; i < n; ++i) logits[(size_t) t * 1024 + i] = nrand() * 2.f;
+ for (auto& v : bias) v = srand_() * 0.3f;
+ Dev<float> dl(logits), db(bias), dw; Dev<int> di; dw.alloc((size_t) T * kPicks); di.alloc((size_t) T * kPicks);
+ route(dl.p, db.p, n, k, T, rs, di.p, dw.p, g_s); CK(cudaStreamSynchronize(g_s));
+ auto gi = di.get(); auto gw = dw.get();
+ int bad_idx = 0; std::vector<double> rr; std::vector<float> gg;
+ for (int t = 0; t < T; ++t) {
+ std::vector<double> sc(n), bs(n);
+ for (int i = 0; i < n; ++i) { const double l = logits[(size_t) t * 1024 + i]; sc[i] = std::sqrt(l > 20 ? l : std::log1p(std::exp(l))); bs[i] = sc[i] + bias[i]; }
+ std::vector<int> ord(n); for (int i = 0; i < n; ++i) ord[i] = i;
+ std::partial_sort(ord.begin(), ord.begin() + k, ord.end(), [&](int a, int b) { return bs[a] > bs[b] || (bs[a] == bs[b] && a < b); });
+ double s = 0; for (int r = 0; r < k; ++r) s += sc[ord[r]];
+ for (int r = 0; r < k; ++r) { bad_idx += gi[(size_t) t * kPicks + r] != ord[r]; rr.push_back(sc[ord[r]] / (s + 1e-20) * rs); gg.push_back(gw[(size_t) t * kPicks + r]); }
+ }
+ char ex[64]; std::snprintf(ex, sizeof ex, "%d / %d picks differ", bad_idx, T * k);
+ report("route (sqrt-softplus)", T, bad_idx ? 1.0 : max_rel(gg, rr), 1e-5, time_us([&] { route(dl.p, db.p, n, k, T, rs, di.p, dw.p, g_s); }), ex);
+}
+
+// ============================================================================================================ compressor
+static void test_compress(int T, int ratio) {
+ const int d = 512; const float eps = 1e-6f;
+ // state pre-filled for the slots before p0; p0 chosen so that the last token of the window completes a group
+ const int p0 = ratio * 3 + (ratio - T);
+ std::vector<float> kv((size_t) T * d), sc((size_t) T * d), kst((size_t) ratio * d), sst((size_t) ratio * d), w(d);
+ for (auto& v : kv) v = nrand(); for (auto& v : sc) v = nrand() * 2.f; for (auto& v : kst) v = nrand(); for (auto& v : sst) v = nrand() * 2.f;
+ for (auto& v : w) v = 0.5f + urand();
+ Dev<float> dkv(kv), dsc(sc), dk(kst), ds(sst), dw(w), dlat; dlat.alloc((size_t) d * 2);
+ compress_win(dkv.p, dsc.p, dk.p, ds.p, ratio, p0, T, d, dw.p, eps, dlat.p, g_s); CK(cudaStreamSynchronize(g_s));
+ auto lat = dlat.get();
+ // reference: slots (p0+t)%ratio overwritten by the window's tokens, group = all `ratio` slots
+ std::vector<float> k2 = kst, s2 = sst;
+ for (int t = 0; t < T; ++t) { const int slot = (p0 + t) % ratio; for (int e = 0; e < d; ++e) { k2[(size_t) slot * d + e] = kv[(size_t) t * d + e]; s2[(size_t) slot * d + e] = sc[(size_t) t * d + e]; } }
+ std::vector<double> rr(d); double ss = 0;
+ for (int e = 0; e < d; ++e) { double m = -1e300; for (int s = 0; s < ratio; ++s) m = std::max(m, (double) s2[(size_t) s * d + e]);
+ double den = 0; for (int s = 0; s < ratio; ++s) den += std::exp(s2[(size_t) s * d + e] - m);
+ double a = 0; for (int s = 0; s < ratio; ++s) a += (double) k2[(size_t) s * d + e] * std::exp(s2[(size_t) s * d + e] - m) / den; rr[e] = a; ss += a * a; }
+ const double inv = 1.0 / std::sqrt(ss / d + eps); for (int e = 0; e < d; ++e) rr[e] = rr[e] * inv * w[e];
+ char nm[32]; std::snprintf(nm, sizeof nm, "compress_win r=%d", ratio);
+ std::vector<float> gg(lat.begin(), lat.begin() + d);
+ report(nm, T, max_rel(gg, rr), 5e-6, time_us([&] { compress_win(dkv.p, dsc.p, dk.p, ds.p, ratio, p0, T, d, dw.p, eps, dlat.p, g_s); }), "d 512, pooling window");
+}
+
+// ============================================================================================================ indexer
+static void test_indexer(int T, int n) {
+ const int heads = 32, hd = 128, topk_k = 512;
+ const size_t rb = kv_row_bytes(kKvFp4p, hd);
+ std::vector<uint8_t> keys((size_t) n * rb, 0);
+ std::vector<float> kdec((size_t) n * hd);
+ for (int p = 0; p < n; ++p) {
+ uint8_t* r = &keys[(size_t) p * rb];
+ for (int j = 0; j < hd / 2; ++j) r[j] = (uint8_t) (rnd() & 0xff);
+ for (int g = 0; g < hd / 32; ++g) r[hd / 2 + g] = (uint8_t) (120 + (rnd() % 5)); // 2^-7 .. 2^-3
+ for (int e = 0; e < hd; ++e) { const int code = (r[e / 2] >> (4 * (e & 1))) & 15; kdec[(size_t) p * hd + e] = e2m1_val(code) * std::ldexp(1.f, (int) r[hd / 2 + e / 32] - 127); }
+ }
+ std::vector<float> q((size_t) T * heads * hd), w((size_t) T * 64, 0.f);
+ for (auto& v : q) v = nrand(); for (int t = 0; t < T; ++t) for (int h = 0; h < heads; ++h) w[(size_t) t * 64 + h] = srand_();
+ Dev<uint8_t> dk(keys); Dev<float> dq(q), dw(w), dout; dout.alloc((size_t) T * n);
+ Rows lim; for (int t = 0; t < kWin; ++t) { lim.n[t] = n; lim.k[t] = topk_k; }
+ index_score(dq.p, dw.p, dk.p, heads, hd, T, n, nullptr, 0, 1, lim, dout.p, n, g_s); CK(cudaStreamSynchronize(g_s));
+ auto sc = dout.get();
+ std::vector<double> rr; std::vector<float> gg;
+ for (int t = 0; t < T; ++t) for (int p = 0; p < n; p += std::max(1, n / 512)) {
+ double s = 0; for (int h = 0; h < heads; ++h) { double a = 0; for (int e = 0; e < hd; ++e) a += (double) q[((size_t) t * heads + h) * hd + e] * kdec[(size_t) p * hd + e]; s += std::max(a, 0.0) * w[(size_t) t * 64 + h]; }
+ rr.push_back(s); gg.push_back(sc[(size_t) t * n + p]);
+ }
+ char ex[96]; std::snprintf(ex, sizeof ex, "%d keys, %.1f MB of keys", n, n * (double) rb / 1e6);
+ const double us = time_us([&] { index_score(dq.p, dw.p, dk.p, heads, hd, T, n, nullptr, 0, 1, lim, dout.p, n, g_s); });
+ report("index_score", T, max_rel(gg, rr), 3e-5, us, ex);
+ // top-512 of the device's own scores: exact set compare
+ Rows tr; for (int t = 0; t < kWin; ++t) { tr.n[t] = n; tr.k[t] = std::min(topk_k, n); }
+ Dev<int> dsel; dsel.alloc((size_t) T * topk_k);
+ topk(dout.p, n, T, tr, nullptr, 0, 1, dsel.p, topk_k, g_s); CK(cudaStreamSynchronize(g_s));
+ auto sel = dsel.get(); int bad = 0;
+ for (int t = 0; t < T; ++t) {
+ std::vector<int> ord(n); for (int i = 0; i < n; ++i) ord[i] = i;
+ std::nth_element(ord.begin(), ord.begin() + topk_k, ord.end(), [&](int a, int b) { return sc[(size_t) t * n + a] > sc[(size_t) t * n + b]; });
+ std::vector<char> in(n, 0); for (int i = 0; i < topk_k; ++i) in[ord[i]] = 1;
+ const float kth = sc[(size_t) t * n + *std::min_element(ord.begin(), ord.begin() + topk_k, [&](int a, int b) { return sc[(size_t) t * n + a] < sc[(size_t) t * n + b]; })];
+ for (int i = 0; i < topk_k; ++i) { const int p = sel[(size_t) t * topk_k + i]; if (p < 0 || p >= n || (!in[p] && sc[(size_t) t * n + p] != kth)) ++bad; }
+ }
+ std::snprintf(ex, sizeof ex, "%d wrong picks of %d", bad, T * topk_k);
+ report("topk 512", T, bad ? 1.0 : 0.0, 0.0, time_us([&] { topk(dout.p, n, T, tr, nullptr, 0, 1, dsel.p, topk_k, g_s); }), ex);
+}
+
+// ============================================================================================================ attention
+static void test_attn(int T, int ncache) {
+ const int heads = 64, d = 512, nwin = 128, nsel = 512;
+ const float scale = 1.f / std::sqrt((float) d);
+ const size_t rb = kv_row_bytes(kKvFp4, d); // 304
+ std::vector<uint8_t> ckv((size_t) ncache * rb, 0);
+ std::vector<float> cdec((size_t) ncache * d);
+ for (int p = 0; p < ncache; ++p) {
+ uint8_t* r = &ckv[(size_t) p * rb];
+ for (int j = 0; j < d / 2; ++j) r[j] = (uint8_t) (rnd() & 0xff);
+ for (int g = 0; g < d / 16; ++g) r[d / 2 + g] = (uint8_t) ((7 << 3) | (rnd() & 7)) - (uint8_t) (rnd() % 3 * 8); // ~1 .. 0.25
+ for (int e = 0; e < d; ++e) { const int code = (r[e / 2] >> (4 * (e & 1))) & 15; cdec[(size_t) p * d + e] = e2m1_val(code) * e4m3_val(r[d / 2 + e / 16]); }
+ }
+ // window buffer: nwin + T - 1 rows (each query sees nwin of them)
+ const int wrows = nwin + T;
+ std::vector<uint16_t> win((size_t) wrows * d); std::vector<float> wdec((size_t) wrows * d);
+ for (size_t i = 0; i < win.size(); ++i) { win[i] = f2bf(nrand()); wdec[i] = bf2f(win[i]); }
+ std::vector<float> q((size_t) T * heads * d), sink(heads);
+ for (auto& v : q) v = nrand(); for (auto& v : sink) v = srand_();
+ std::vector<int> sel((size_t) T * nsel);
+ for (int t = 0; t < T; ++t) for (int i = 0; i < nsel; ++i) sel[(size_t) t * nsel + i] = (int) (rnd() % ncache);
+ AttnRows ar; for (int t = 0; t < kWin; ++t) { ar.win0[t] = t < T ? t + 1 : 0; ar.nwin[t] = nwin; ar.nsel[t] = nsel; }
+ Dev<uint8_t> dck(ckv); Dev<uint16_t> dwin(win); Dev<float> dq(q), dsink(sink), dout; Dev<int> dsel(sel); dout.alloc((size_t) T * heads * d);
+ Dev<uint16_t> dscr; dscr.alloc((size_t) kAttnScratchT * (nsel + 8) * d);
+ AttnScratch scr; scr.rows = dscr.p; scr.max_t = kAttnScratchT; scr.cap = nsel + 8;
+ sparse_attn(dq.p, dwin.p, dck.p, dsel.p, nsel, ar, dsink.p, heads, d, T, scale, dout.p, g_s, kKvFp4, 0, &scr); CK(cudaStreamSynchronize(g_s));
+ auto o = dout.get(); std::vector<double> rr; std::vector<float> gg;
+ for (int t = 0; t < T; t += std::max(1, T / 2)) for (int h = 0; h < heads; h += 7) {
+ std::vector<double> s; std::vector<const float*> rows;
+ for (int j = 0; j < nwin; ++j) rows.push_back(&wdec[(size_t) (ar.win0[t] + j) * d]);
+ for (int i = 0; i < nsel; ++i) rows.push_back(&cdec[(size_t) sel[(size_t) t * nsel + i] * d]);
+ double m = sink[h];
+ for (auto* rw : rows) { double a = 0; for (int e = 0; e < d; ++e) a += (double) q[((size_t) t * heads + h) * d + e] * rw[e]; s.push_back(a * scale); m = std::max(m, s.back()); }
+ double den = std::exp(sink[h] - m); for (double v : s) den += std::exp(v - m);
+ std::vector<double> acc(d, 0.0);
+ for (size_t j = 0; j < rows.size(); ++j) { const double p = std::exp(s[j] - m) / den; for (int e = 0; e < d; ++e) acc[e] += p * rows[j][e]; }
+ for (int e = 0; e < d; ++e) { rr.push_back(acc[e]); gg.push_back(o[((size_t) t * heads + h) * d + e]); }
+ }
+ char ex[96]; std::snprintf(ex, sizeof ex, "64 heads x 512, 128 win + 512 sel, cache %d rows", ncache);
+ report("sparse_attn (fp4 ckv)", T, max_rel(gg, rr), 1e-4, time_us([&] { sparse_attn(dq.p, dwin.p, dck.p, dsel.p, nsel, ar, dsink.p, heads, d, T, scale, dout.p, g_s, kKvFp4, 0, &scr); }), ex);
+ // the packed-rows kernel (no scratch): used from 8 queries on, timed here at this T for comparison
+ sparse_attn(dq.p, dwin.p, dck.p, dsel.p, nsel, ar, dsink.p, heads, d, T, scale, dout.p, g_s, kKvFp4, 0, nullptr); CK(cudaStreamSynchronize(g_s));
+ { auto o2 = dout.get(); std::vector<float> g2; for (int t = 0; t < T; t += std::max(1, T / 2)) for (int h = 0; h < heads; h += 7) for (int e = 0; e < d; ++e) g2.push_back(o2[((size_t) t * heads + h) * d + e]);
+ report("sparse_attn (packed)", T, max_rel(g2, rr), 1e-4, time_us([&] { sparse_attn(dq.p, dwin.p, dck.p, dsel.p, nsel, ar, dsink.p, heads, d, T, scale, dout.p, g_s, kKvFp4, 0, nullptr); })); }
+}
+
+// ============================================================================================================ Engram
+static void test_engram(int T) {
+ const int cols = 24, hd = 256, hc = 4, dim = 5120; const float eps = 1e-6f;
+ // dequant: n rows FP8 E4M3 + E8M0 scales
+ const int n = T * cols;
+ std::vector<uint8_t> rows((size_t) n * hd), scales((size_t) n * (hd / 32));
+ for (auto& v : rows) { do v = (uint8_t) (rnd() & 0xff); while ((v & 0x7f) == 0x7f); }
+ for (auto& v : scales) v = (uint8_t) (115 + rnd() % 8);
+ Dev<uint8_t> dr(rows), dsc(scales); Dev<float> dout; dout.alloc((size_t) n * hd);
+ engram_dequant(dr.p, dsc.p, n, hd, dout.p, g_s); CK(cudaStreamSynchronize(g_s));
+ auto o = dout.get(); size_t bad = 0;
+ for (int r = 0; r < n; ++r) for (int e = 0; e < hd; ++e) { const float ref = e4m3_val(rows[(size_t) r * hd + e]) * std::ldexp(1.f, (int) scales[(size_t) r * (hd / 32) + e / 32] - 127); if (std::memcmp(&ref, &o[(size_t) r * hd + e], 4) != 0) ++bad; }
+ char ex[64]; std::snprintf(ex, sizeof ex, "%zu of %zu values differ (bit exact)", bad, (size_t) n * hd);
+ report("engram_dequant", T, bad ? 1.0 : 0.0, 0.0, time_us([&] { engram_dequant(dr.p, dsc.p, n, hd, dout.p, g_s); }), ex);
+ // gate
+ std::vector<float> h((size_t) T * hc * dim), kv((size_t) T * (hc + 1) * dim), w((size_t) hc * dim);
+ for (auto& v : h) v = nrand(); for (auto& v : kv) v = nrand(); for (auto& v : w) v = 0.5f + urand();
+ Dev<float> dh(h), dkv(kv), dw(w);
+ engram_gate(dh.p, dkv.p, dw.p, hc, dim, T, eps, g_s); CK(cudaStreamSynchronize(g_s));
+ auto go = dh.get(); std::vector<double> rr; std::vector<float> gg;
+ for (int t = 0; t < T; ++t) for (int k = 0; k < hc; ++k) {
+ const float* hk = &h[((size_t) t * hc + k) * dim]; const float* key = &kv[(size_t) t * (hc + 1) * dim + (size_t) k * dim]; const float* val = &kv[(size_t) t * (hc + 1) * dim + (size_t) hc * dim];
+ double sh = 0, sk = 0, sd = 0; for (int e = 0; e < dim; ++e) { sh += (double) hk[e] * hk[e]; sk += (double) key[e] * key[e]; sd += (double) hk[e] * w[(size_t) k * dim + e] * key[e]; }
+ const double rstd = 1.0 / std::sqrt(sh / dim + eps) / std::sqrt(sk / dim + eps);
+ const double dot = sd * rstd / std::sqrt((double) dim); const double sq = std::copysign(std::sqrt(std::max(std::fabs(dot), 1e-6)), dot); const double gate = 1.0 / (1.0 + std::exp(-sq));
+ for (int e = 0; e < dim; ++e) { rr.push_back(hk[e] + gate * val[e]); gg.push_back(go[((size_t) t * hc + k) * dim + e]); }
+ }
+ Dev<float> dh2(h);
+ report("engram_gate", T, max_rel(gg, rr), 2e-6, time_us([&] { engram_gate(dh2.p, dkv.p, dw.p, hc, dim, T, eps, g_s); }), "(timing re-applies in place)");
+ // the n-gram row gather: 24 hash-picked rows of 256 per token = the host/SSD side; on the device only the gather of
+ // x rows (gather_rows) - T x 5120 floats
+ Gather gt; for (int i = 0; i < kWin * kPicks; ++i) gt.src[i] = (int) (rnd() % T);
+ Dev<float> dx; { std::vector<float> x((size_t) T * dim); for (auto& v : x) v = nrand(); dx.set(x); } Dev<float> dg; dg.alloc((size_t) T * kPicks * dim);
+ gather_rows(dx.p, gt, T * kPicks > kWin * kPicks ? kWin * kPicks : T * kPicks, dim, dg.p, g_s); CK(cudaStreamSynchronize(g_s));
+ auto go2 = dg.get(); size_t badg = 0; const int ng = std::min(T * kPicks, kWin * kPicks);
+ std::vector<float> xs = dx.get();
+ for (int r = 0; r < ng; ++r) for (int c = 0; c < dim; ++c) badg += go2[(size_t) r * dim + c] != xs[(size_t) gt.src[r] * dim + c];
+ std::snprintf(ex, sizeof ex, "%zu mismatches, %d rows", badg, ng);
+ report("gather_rows", T, badg ? 1.0 : 0.0, 0.0, time_us([&] { gather_rows(dx.p, gt, ng, dim, dg.p, g_s); }), ex);
+}
+
+// ============================================================================================================ swiglu + combine
+static void test_swiglu(int T) {
+ const int n = T * 2304 * 6; const float lim = 10.f;
+ std::vector<float> g(n), u(n); for (auto& v : g) v = nrand() * 6.f; for (auto& v : u) v = nrand() * 6.f;
+ Dev<float> dg(g), du(u), dh; dh.alloc(n);
+ swiglu(dg.p, du.p, n, lim, dh.p, g_s); CK(cudaStreamSynchronize(g_s));
+ auto h = dh.get(); std::vector<double> rr; for (int i = 0; i < n; ++i) { const double gv = std::min((double) g[i], (double) lim), uv = std::min(std::max((double) u[i], -(double) lim), (double) lim); rr.push_back(gv / (1.0 + std::exp(-gv)) * uv); }
+ report("swiglu clamp 10", T, max_rel(h, rr), 3e-6, time_us([&] { swiglu(dg.p, du.p, n, lim, dh.p, g_s); }), "T x 6 experts x 2304");
+}
+
+int main(int argc, char** argv) {
+ std::vector<int> Ts = {1, 2, 3, 4, 5};
+ std::string only; int ctx = 8192;
+ for (int i = 1; i < argc; ++i) {
+ std::string a = argv[i];
+ if (a == "--tokens" && i + 1 < argc) { Ts.clear(); for (const char* p = argv[++i]; *p;) { Ts.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
+ else if (a == "--only" && i + 1 < argc) only = std::string(",") + argv[++i] + ",";
+ else if (a == "--ctx" && i + 1 < argc) ctx = std::atoi(argv[++i]);
+ else if (a == "--reps" && i + 1 < argc) g_reps = std::atoi(argv[++i]);
+ else if (a == "--dev" && i + 1 < argc) CK(cudaSetDevice(std::atoi(argv[++i])));
+ }
+ cudaDeviceProp prop; int dev; CK(cudaGetDevice(&dev)); CK(cudaGetDeviceProperties(&prop, dev));
+ std::printf("device %s sm_%d%d; kernels: src/dsv41/ds4_kernels.cu unmodified; graph-replayed timings (no launch gaps)\n", prop.name, prop.major, prop.minor);
+ CK(cudaStreamCreate(&g_s));
+ const int ncache = ctx / 4; // compressed rows at ratio 4
+ auto want = [&](const char* n) { return only.empty() || only.find(std::string(",") + n + ",") != std::string::npos; };
+ for (int T : Ts) {
+ if (want("mhc")) test_mhc(T);
+ if (want("router")) test_router(T);
+ if (want("compress")) { test_compress(T, 4); test_compress(T, 128); }
+ if (want("indexer")) test_indexer(T, ncache);
+ if (want("attn")) test_attn(T, ncache);
+ if (want("engram")) test_engram(T);
+ if (want("swiglu")) test_swiglu(T);
+ std::printf("\n");
+ }
+ std::printf("%s (%d failures)\n", g_fails ? "FAILED" : "ALL PARITY OK", g_fails);
+ return g_fails ? 1 : 0;
+}