Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41 cudalab: split-KV sparse attention (2.9x at T=1), CPU-kernel cross-check of the trellis GEMV, build for the repo CPU kernels

Niko12211c20a5967555
diff --git a/tools/dsv41_lab/attn_split.cuh b/tools/dsv41_lab/attn_split.cuh
new file mode 100644
index 000000000..2ddbf95b6
--- /dev/null
+++ b/tools/dsv41_lab/attn_split.cuh
@@ -0,0 +1,202 @@
+// tools/dsv41_lab/attn_split.cuh - split-KV ("flash-decoding") version of DeepSeek's sparse attention for a few queries
+// (decode T = 1, DSpark verify T = 2..5): 64 heads x d 512, MQA (K = V), window rows (BF16) + selected compressed rows (kKvFp4:
+// E2M1 codes + E4M3 scales per 16) + a learned sink logit per head. Same inputs and outputs as strata::ds4::sparse_attn
+// (AttnRows gen 0, ckv kKvFp4); the shipped kernel runs one 256-thread block per (head, query) = 64 blocks on 48 SMs and takes
+// ~125 us per query; this one splits the keys over R-row chunks and groups G heads per block, which share the decoded rows.
+// phase 1: block (head group, chunk, query): decode R rows to BF16 in shared memory (exact: window rows are BF16, packed
+// rows have <= 7 significant bits), scores for G heads (warp per row, lanes 16 values, shuffle reduction), local
+// softmax, partial output (thread = 2 output values) -> scratch {m, l, o[512]} per (query, head, chunk)
+// phase 2: block (head, query): merge the chunks and the sink.
+// Plain C++ (no PTX), shuffles and shared memory only: the HIP port is a rename.
+#pragma once
+#include "../../src/dsv41/ds4_kernels.cuh"
+#include "../../src/dsv41/ds4_numerics.cuh"
+#include <cuda_fp16.h>
+
+namespace strata::ds4lab {
+
+using strata::ds4::AttnRows;
+
+__device__ __forceinline__ float bf2f_(uint16_t b) { return __uint_as_float((uint32_t) b << 16); }
+__device__ __forceinline__ uint16_t f2bf_exact(float f) { return (uint16_t) (__float_as_uint(f) >> 16); } // exact values only
+__device__ __forceinline__ float e2m1_x2p14(uint32_t c) {
+ return __half2float(__ushort_as_half((unsigned short) (((c & 8u) << 12) | ((c & 7u) << 9))));
+}
+__device__ __forceinline__ float wsum(float v) {
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o);
+ return v;
+}
+__device__ __forceinline__ float wmax(float v) {
+#pragma unroll
+ for (int o = 16; o > 0; o >>= 1) v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, o));
+ return v;
+}
+
+// d = 512 only (a thread owns 2 output values, a lane 16 values of a row)
+template <int G, int R>
+__global__ void __launch_bounds__(256) attn_part(const float* __restrict__ q, const uint16_t* __restrict__ win,
+ const uint8_t* __restrict__ ckv, size_t ckv_row_bytes, const int* __restrict__ sel,
+ int sel_stride, AttnRows ar, int heads, float scale, float* __restrict__ part,
+ float* __restrict__ ml, int nsplit) {
+ constexpr int D = 512;
+ __shared__ __align__(16) uint16_t rows[R][D];
+ __shared__ float sc[G][R];
+ __shared__ float4 pr[R][(G + 3) / 4];
+ __shared__ float mloc[G], lloc[G];
+ const int hg = blockIdx.x, chunk = blockIdx.y, t = blockIdx.z;
+ const int n_win = ar.nwin[t], n_sel = ar.nsel[t], total = n_win + n_sel;
+ const int j0 = chunk * R;
+ if (j0 >= total) return;
+ const int nr = min(R, total - j0);
+ const int lane = threadIdx.x & 31, wp = threadIdx.x >> 5;
+ // q of this block's heads: lane's 16 values per head in registers (issued first: its latency overlaps the row loads)
+ float qv[G][16];
+#pragma unroll
+ for (int g = 0; g < G; ++g) {
+ const float4* qp = (const float4*) (q + ((size_t) t * heads + hg * G + g) * D + lane * 16);
+#pragma unroll
+ for (int i = 0; i < 4; ++i) { const float4 v = qp[i]; qv[g][4 * i] = v.x; qv[g][4 * i + 1] = v.y; qv[g][4 * i + 2] = v.z; qv[g][4 * i + 3] = v.w; }
+ }
+ // ---- stage rows: warp per row, lane = 16 values; all of a warp's row loads are issued before any is converted
+ constexpr int RPW = (R + 7) / 8;
+ uint4 wa[RPW][2];
+ uint2 cwv[RPW];
+ uint32_t sbv[RPW];
+ int rowoff[RPW];
+#pragma unroll
+ for (int i = 0; i < RPW; ++i) {
+ const int jj = wp + 8 * i, j = j0 + jj;
+ rowoff[i] = -1;
+ if (jj < nr) {
+ if (j < n_win) {
+ const uint4* src = (const uint4*) (win + (size_t) (ar.win0[t] + j) * D + lane * 16);
+ wa[i][0] = src[0];
+ wa[i][1] = src[1];
+ } else {
+ rowoff[i] = sel[(size_t) t * sel_stride + (j - n_win)];
+ }
+ }
+ }
+#pragma unroll
+ for (int i = 0; i < RPW; ++i)
+ if (rowoff[i] >= 0) {
+ const uint8_t* r = ckv + (size_t) rowoff[i] * ckv_row_bytes;
+ cwv[i] = *(const uint2*) (r + lane * 8);
+ sbv[i] = r[D / 2 + lane];
+ }
+#pragma unroll
+ for (int i = 0; i < RPW; ++i) {
+ const int jj = wp + 8 * i;
+ if (jj >= nr) continue;
+ uint4* dst = (uint4*) &rows[jj][lane * 16];
+ if (rowoff[i] < 0) {
+ dst[0] = wa[i][0];
+ dst[1] = wa[i][1];
+ } else {
+ const float s = strata::ds4::e4m3_value((uint8_t) sbv[i]) * 16384.0f; // (the nibbles decode to value * 2^-14)
+ uint32_t o[8];
+#pragma unroll
+ for (int w = 0; w < 2; ++w) {
+ const uint32_t c = w ? cwv[i].y : cwv[i].x;
+#pragma unroll
+ for (int x = 0; x < 4; ++x) {
+ const uint32_t b = c >> (8 * x);
+ const uint16_t lo = f2bf_exact(e2m1_x2p14(b) * s), hi = f2bf_exact(e2m1_x2p14(b >> 4) * s);
+ o[4 * w + x] = (uint32_t) lo | ((uint32_t) hi << 16);
+ }
+ }
+ dst[0] = make_uint4(o[0], o[1], o[2], o[3]);
+ dst[1] = make_uint4(o[4], o[5], o[6], o[7]);
+ }
+ }
+ __syncthreads();
+ // ---- scores
+ for (int jj = wp; jj < nr; jj += 8) {
+ const uint4 a = *(const uint4*) &rows[jj][lane * 16], b = *(const uint4*) &rows[jj][lane * 16 + 8];
+ float kv[16];
+ const uint32_t w[8] = {a.x, a.y, a.z, a.w, b.x, b.y, b.z, b.w};
+#pragma unroll
+ for (int i = 0; i < 8; ++i) { kv[2 * i] = __uint_as_float(w[i] << 16); kv[2 * i + 1] = __uint_as_float(w[i] & 0xffff0000u); }
+#pragma unroll
+ for (int g = 0; g < G; ++g) {
+ float s = 0.f;
+#pragma unroll
+ for (int i = 0; i < 16; ++i) s += qv[g][i] * kv[i];
+ s = wsum(s);
+ if (lane == 0) sc[g][jj] = s * scale;
+ }
+ }
+ __syncthreads();
+ // ---- local softmax: warp g handles head g
+ if (wp < G) {
+ float m = -INFINITY;
+ for (int j = lane; j < nr; j += 32) m = fmaxf(m, sc[wp][j]);
+ m = wmax(m);
+ float l = 0.f;
+ for (int j = lane; j < nr; j += 32) { const float p = __expf(sc[wp][j] - m); sc[wp][j] = p; l += p; }
+ l = wsum(l);
+ if (lane == 0) { mloc[wp] = m; lloc[wp] = l; }
+ }
+ __syncthreads();
+ for (int i = threadIdx.x; i < nr * G; i += 256) { const int j = i / G, g = i % G; ((float*) &pr[j][0])[g] = sc[g][j]; }
+ __syncthreads();
+ // ---- partial output: thread = values 2 tid, 2 tid + 1 of every head
+ float o0[G], o1[G];
+#pragma unroll
+ for (int g = 0; g < G; ++g) o0[g] = o1[g] = 0.f;
+ const int c = threadIdx.x * 2;
+ for (int j = 0; j < nr; ++j) {
+ const uint32_t kw = *(const uint32_t*) &rows[j][c];
+ const float k0 = __uint_as_float(kw << 16), k1 = __uint_as_float(kw & 0xffff0000u);
+ const float* p = (const float*) &pr[j][0];
+#pragma unroll
+ for (int g = 0; g < G; ++g) { o0[g] += p[g] * k0; o1[g] += p[g] * k1; }
+ }
+#pragma unroll
+ for (int g = 0; g < G; ++g) {
+ const size_t bi = ((size_t) t * heads + hg * G + g) * nsplit + chunk;
+ *(float2*) (part + bi * D + c) = make_float2(o0[g], o1[g]);
+ if (threadIdx.x == 0) { ml[bi * 2] = mloc[g]; ml[bi * 2 + 1] = lloc[g]; }
+ }
+}
+
+__global__ void __launch_bounds__(256) attn_merge(const float* __restrict__ part, const float* __restrict__ ml, const float* __restrict__ sink,
+ int heads, int nsplit, int R, AttnRows ar, float* __restrict__ out) {
+ constexpr int D = 512;
+ const int h = blockIdx.x, t = blockIdx.y;
+ const int total = ar.nwin[t] + ar.nsel[t];
+ const int ns = (total + R - 1) / R;
+ const size_t base = ((size_t) t * heads + h) * nsplit;
+ float M = sink[h];
+ for (int s = 0; s < ns; ++s) M = fmaxf(M, ml[(base + s) * 2]);
+ float den = __expf(sink[h] - M);
+ for (int s = 0; s < ns; ++s) den += ml[(base + s) * 2 + 1] * __expf(ml[(base + s) * 2] - M);
+ const int c = threadIdx.x * 2;
+ float a0 = 0.f, a1 = 0.f;
+ for (int s = 0; s < ns; ++s) {
+ const float w = __expf(ml[(base + s) * 2] - M);
+ const float2 v = *(const float2*) (part + (base + s) * D + c);
+ a0 += w * v.x;
+ a1 += w * v.y;
+ }
+ *(float2*) (out + ((size_t) t * heads + h) * D + c) = make_float2(a0 / den, a1 / den);
+}
+
+struct AttnSplitScratch {
+ float* part = nullptr; // [T][heads][nsplit][512]
+ float* ml = nullptr; // [T][heads][nsplit][2]
+ int nsplit = 0, max_t = 0;
+};
+
+// total_max = the largest nwin + nsel of the T queries
+template <int G, int R>
+void sparse_attn_split(const float* q, const uint16_t* win, const uint8_t* ckv, size_t ckv_row_bytes, const int* sel, int sel_stride,
+ const AttnRows& ar, const float* sink, int heads, int T, float scale, float* out, const AttnSplitScratch& sc,
+ int total_max, cudaStream_t s) {
+ const int nsplit = (total_max + R - 1) / R;
+ attn_part<G, R><<<dim3(heads / G, nsplit, T), 256, 0, s>>>(q, win, ckv, ckv_row_bytes, sel, sel_stride, ar, heads, scale, sc.part, sc.ml, sc.nsplit);
+ attn_merge<<<dim3(heads, T), 256, 0, s>>>(sc.part, sc.ml, sink, heads, sc.nsplit, R, ar, out);
+}
+
+} // namespace strata::ds4lab
diff --git a/tools/dsv41_lab/build.sh b/tools/dsv41_lab/build.sh
index 5007bbf99..f52efd269 100644
--- a/tools/dsv41_lab/build.sh
+++ b/tools/dsv41_lab/build.sh
@@ -1,11 +1,16 @@
#!/bin/bash
-# build the trellis kernel lab on a CUDA box: tools/dsv41_lab/build.sh [arch, default sm_86]
+# build the kernel labs on a CUDA box: tools/dsv41_lab/build.sh [arch, default sm_86]
set -e
cd "$(dirname "$0")"
ARCH=${1:-sm_86}
NVCC=${NVCC:-/usr/local/cuda-12.9/bin/nvcc}
mkdir -p out
-g++ -O2 -std=c++17 -I../../third_party/ggml -c ../trellis/tqenc.cpp -o out/tqenc.o
-$NVCC -O3 -std=c++17 -arch=$ARCH -lineinfo -o out/tqk_lab tqk_lab.cu out/tqenc.o -lpthread
+CPU=../../src/kernels/cpu
+INC="-I../../include -I../../third_party/ggml -I$CPU"
+g++ -O2 -std=c++17 $INC -c ../trellis/tqenc.cpp -o out/tqenc.o
+g++ -O2 -std=c++17 $INC -c $CPU/tq_cpu.cpp -o out/tq_cpu.o
+g++ -O2 -std=c++17 $INC -mavx2 -mfma -mf16c -c $CPU/tq_avx2.cpp -o out/tq_avx2.o
+g++ -O2 -std=c++17 $INC -mavx512f -mavx512bw -mavx512vl -mavx512vnni -mavx512vbmi -mavx2 -mfma -mf16c -c $CPU/tq_avx512.cpp -o out/tq_avx512.o
+$NVCC -O3 -std=c++17 -arch=$ARCH -lineinfo -I../../include -o out/tqk_lab tqk_lab.cu out/tqenc.o out/tq_cpu.o out/tq_avx2.o out/tq_avx512.o -lpthread
# DeepSeek kernel lab: links src/dsv41/ds4_kernels.cu unmodified
$NVCC -O3 -std=c++17 -arch=$ARCH -lineinfo -I../../include -o out/ds4_lab ds4_lab.cu ../../src/dsv41/ds4_kernels.cu
diff --git a/tools/dsv41_lab/ds4_lab.cu b/tools/dsv41_lab/ds4_lab.cu
index 42f4e7eae..2f18e1b39 100644
--- a/tools/dsv41_lab/ds4_lab.cu
+++ b/tools/dsv41_lab/ds4_lab.cu
@@ -5,6 +5,7 @@
// math (DeepSeek model.py), not from the kernels.
// ds4_lab [--tokens 1,2,3,4,5] [--only name,name] [--ctx 8192] [--reps 100]
#include "../../src/dsv41/ds4_kernels.cuh"
+#include "attn_split.cuh"
#include <algorithm>
#include <cmath>
@@ -284,6 +285,25 @@ static void test_attn(int T, int ncache) {
}
char ex[96]; std::snprintf(ex, sizeof ex, "64 heads x 512, 128 win + 512 sel, cache %d rows", ncache);
report("sparse_attn (fp4 ckv)", T, max_rel(gg, rr), 1e-4, time_us([&] { sparse_attn(dq.p, dwin.p, dck.p, dsel.p, nsel, ar, dsink.p, heads, d, T, scale, dout.p, g_s, kKvFp4, 0, &scr); }), ex);
+ // split-KV version (tools/dsv41_lab/attn_split.cuh), same inputs
+ {
+ const int total = nwin + nsel;
+ auto run = [&](const char* nm, auto launch_fn, int R) {
+ const int nsplit = (total + R - 1) / R;
+ Dev<float> part, ml; part.alloc((size_t) T * heads * nsplit * d); ml.alloc((size_t) T * heads * nsplit * 2);
+ strata::ds4lab::AttnSplitScratch ss; ss.part = part.p; ss.ml = ml.p; ss.nsplit = nsplit; ss.max_t = T;
+ Dev<float> o3; o3.alloc((size_t) T * heads * d);
+ auto go = [&] { launch_fn(dq.p, dwin.p, dck.p, rb, dsel.p, nsel, ar, dsink.p, heads, T, scale, o3.p, ss, total, g_s); };
+ go(); CK(cudaStreamSynchronize(g_s)); CK(cudaGetLastError());
+ auto oo = o3.get(); std::vector<float> g3;
+ for (int t = 0; t < T; t += std::max(1, T / 2)) for (int h = 0; h < heads; h += 7) for (int e = 0; e < d; ++e) g3.push_back(oo[((size_t) t * heads + h) * d + e]);
+ report(nm, T, max_rel(g3, rr), 1e-4, time_us(go), ex);
+ };
+ run("attn_split G4 R40", [](auto... a) { strata::ds4lab::sparse_attn_split<4, 40>(a...); }, 40);
+ run("attn_split G8 R40", [](auto... a) { strata::ds4lab::sparse_attn_split<8, 40>(a...); }, 40);
+ run("attn_split G4 R20", [](auto... a) { strata::ds4lab::sparse_attn_split<4, 20>(a...); }, 20);
+ run("attn_split G2 R32", [](auto... a) { strata::ds4lab::sparse_attn_split<2, 32>(a...); }, 32);
+ }
// the packed-rows kernel (no scratch): used from 8 queries on, timed here at this T for comparison
sparse_attn(dq.p, dwin.p, dck.p, dsel.p, nsel, ar, dsink.p, heads, d, T, scale, dout.p, g_s, kKvFp4, 0, nullptr); CK(cudaStreamSynchronize(g_s));
{ auto o2 = dout.get(); std::vector<float> g2; for (int t = 0; t < T; t += std::max(1, T / 2)) for (int h = 0; h < heads; h += 7) for (int e = 0; e < d; ++e) g2.push_back(o2[((size_t) t * heads + h) * d + e]);
diff --git a/tools/dsv41_lab/tqk_lab.cu b/tools/dsv41_lab/tqk_lab.cu
index d0a4fe2b0..d2938fa8e 100644
--- a/tools/dsv41_lab/tqk_lab.cu
+++ b/tools/dsv41_lab/tqk_lab.cu
@@ -6,6 +6,7 @@
#include "tqk_gemv3.cuh"
#include "tqk_repack.hpp"
#include "../trellis/tqenc.hpp"
+#include "strata/kernels/cpu/tq_cpu.hpp"
#include <algorithm>
#include <cmath>
@@ -168,24 +169,51 @@ int main(int argc, char** argv) {
const int rows = 256, in = 768, nblk = in / 128;
std::vector<uint8_t> mw; fill_random(mw, K, (int64_t) rows * nblk);
std::vector<float> dec((size_t) rows * in); tqenc::decode_blocks(type, mw.data(), (int64_t) rows * nblk, dec.data());
- std::vector<float> x((size_t) nb * in); for (auto& v : x) v = frnd();
- std::vector<double> ref((size_t) nb * rows), refabs((size_t) nb * rows);
- for (int c = 0; c < nb; ++c) for (int r = 0; r < rows; ++r) { double s = 0, sa = 0; for (int i = 0; i < in; ++i) { const double t = (double) dec[(size_t) r * in + i] * x[(size_t) c * in + i]; s += t; sa += std::fabs(t); } ref[(size_t) c * rows + r] = s; refabs[(size_t) c * rows + r] = sa; }
uint8_t* mwd; CK(cudaMalloc(&mwd, mw.size())); CK(cudaMemcpy(mwd, mw.data(), mw.size(), cudaMemcpyHostToDevice));
- float* xd = dev_f32(x); float* yd; CK(cudaMalloc(&yd, (size_t) nb * rows * 4));
const KFn kf = pick_k(K, nb, variant); const uint64_t* scd = nullptr; uint8_t* wd = mwd;
if (kf.tql) { std::vector<uint32_t> words; std::vector<uint64_t> sc; repack_tql(K, mw.data(), rows, nblk, words, sc);
CK(cudaMalloc(&wd, words.size() * 4)); CK(cudaMemcpy(wd, words.data(), words.size() * 4, cudaMemcpyHostToDevice));
uint64_t* t; CK(cudaMalloc(&t, sc.size() * 8)); CK(cudaMemcpy(t, sc.data(), sc.size() * 8, cudaMemcpyHostToDevice)); scd = t; }
- if (kf.tql) { const size_t n = (size_t) nb * (in / 128) * 32; uint4* xb; CK(cudaMalloc(&xb, n * 16)); prep_b<<<(int) ((n + 255) / 256), 256>>>(xd, in, nb, in, xb); CK(cudaDeviceSynchronize()); xd = (float*) xb; }
- Job j{wd, xd, yd, rows, in, rows, scd, in}; Job* jd; CK(cudaMalloc(&jd, sizeof(Job))); CK(cudaMemcpy(jd, &j, sizeof(Job), cudaMemcpyHostToDevice));
- launch(kf, jd, 1, rows, in); CK(cudaGetLastError());
- std::vector<float> y((size_t) nb * rows); CK(cudaMemcpy(y.data(), yd, y.size() * 4, cudaMemcpyDeviceToHost));
+ auto gpu_run = [&](const std::vector<float>& x) {
+ float* xd = dev_f32(x); float* yd; CK(cudaMalloc(&yd, (size_t) nb * rows * 4));
+ float* xj = xd;
+ if (kf.tql) { const size_t n = (size_t) nb * (in / 128) * 32; uint4* xb; CK(cudaMalloc(&xb, n * 16)); prep_b<<<(int) ((n + 255) / 256), 256>>>(xd, in, nb, in, xb); CK(cudaDeviceSynchronize()); xj = (float*) xb; }
+ Job j{wd, xj, yd, rows, in, rows, scd, in}; Job* jd; CK(cudaMalloc(&jd, sizeof(Job))); CK(cudaMemcpy(jd, &j, sizeof(Job), cudaMemcpyHostToDevice));
+ launch(kf, jd, 1, rows, in); CK(cudaGetLastError());
+ std::vector<float> y((size_t) nb * rows); CK(cudaMemcpy(y.data(), yd, y.size() * 4, cudaMemcpyDeviceToHost));
+ cudaFree(xd); cudaFree(yd); cudaFree(jd); if (kf.tql) cudaFree(xj);
+ return y;
+ };
+ // (1) random F32 activations vs a double reference on the decoded weights
+ std::vector<float> x((size_t) nb * in); for (auto& v : x) v = frnd();
+ std::vector<double> ref((size_t) nb * rows), refabs((size_t) nb * rows);
+ for (int c = 0; c < nb; ++c) for (int r = 0; r < rows; ++r) { double s = 0, sa = 0; for (int i = 0; i < in; ++i) { const double t = (double) dec[(size_t) r * in + i] * x[(size_t) c * in + i]; s += t; sa += std::fabs(t); } ref[(size_t) c * rows + r] = s; refabs[(size_t) c * rows + r] = sa; }
+ std::vector<float> y = gpu_run(x);
double worst = 0; for (size_t i = 0; i < y.size(); ++i) worst = std::max(worst, std::fabs(y[i] - ref[i]) / (refabs[i] + 1e-30));
const bool ok = worst < 2e-6;
- std::printf("gemv nb=%d: max |y - ref| / sum|w x| = %.2e %s\n", nb, worst, ok ? "ok" : "FAIL");
- fails += !ok;
- cudaFree(mwd); cudaFree(xd); cudaFree(yd); cudaFree(jd);
+ // (2) the CPU kernel of the repo (tq_cpu scalar and AVX2, Q8_0 activations): same weights; the GPU runs on the dequantised Q8_0 activations
+ std::vector<uint8_t> q8((size_t) nb * (in / 32) * 34); std::vector<float> xq((size_t) nb * in);
+ for (int c = 0; c < nb; ++c) for (int b = 0; b < in / 32; ++b) {
+ float amax = 0; for (int i = 0; i < 32; ++i) amax = std::max(amax, std::fabs(x[(size_t) c * in + b * 32 + i]));
+ const uint16_t dh = tqenc::float_to_half(amax / 127.f); const float df = amax / 127.f, id = df ? 1.f / df : 0.f, dr = tqenc::half_to_float(dh);
+ uint8_t* blk = &q8[((size_t) c * (in / 32) + b) * 34]; std::memcpy(blk, &dh, 2);
+ for (int i = 0; i < 32; ++i) { const int qv = (int) std::lround(x[(size_t) c * in + b * 32 + i] * id); blk[2 + i] = (uint8_t) (int8_t) qv; xq[(size_t) c * in + b * 32 + i] = qv * dr; }
+ }
+ std::vector<float> ycpu_s((size_t) nb * rows), ycpu_a((size_t) nb * rows);
+ const bool have_avx2 = strata::kernels::cpu::tq_isa_ok(strata::kernels::cpu::TqIsa::avx2);
+ { const void* act[8]; float* os[8]; float* oa[8];
+ for (int c = 0; c < nb; ++c) { act[c] = &q8[(size_t) c * (in / 32) * 34]; os[c] = &ycpu_s[(size_t) c * rows]; oa[c] = &ycpu_a[(size_t) c * rows]; }
+ const size_t row_bytes = (size_t) nblk * bsz;
+ strata::kernels::cpu::tq_rows(strata::kernels::cpu::TqIsa::scalar, type, mw.data(), row_bytes, in, act, nb, os, 0, rows);
+ if (have_avx2) strata::kernels::cpu::tq_rows(strata::kernels::cpu::TqIsa::avx2, type, mw.data(), row_bytes, in, act, nb, oa, 0, rows); }
+ std::vector<float> yq = gpu_run(xq);
+ std::vector<double> refq((size_t) nb * rows);
+ for (int c = 0; c < nb; ++c) for (int r = 0; r < rows; ++r) { double s = 0, sa = 0; for (int i = 0; i < in; ++i) { const double t = (double) dec[(size_t) r * in + i] * xq[(size_t) c * in + i]; s += t; sa += std::fabs(t); } refq[(size_t) c * rows + r] = s; refabs[(size_t) c * rows + r] = sa; }
+ double w_gc = 0, w_cr = 0, w_ac = 0;
+ for (size_t i = 0; i < yq.size(); ++i) { const double den = refabs[i] + 1e-30; w_gc = std::max(w_gc, std::fabs(yq[i] - ycpu_s[i]) / den); w_cr = std::max(w_cr, std::fabs(ycpu_s[i] - refq[i]) / den); if (have_avx2) w_ac = std::max(w_ac, std::fabs(ycpu_a[i] - ycpu_s[i]) / den); }
+ std::printf("gemv nb=%d: GPU vs double ref %.2e %s | vs CPU kernel of the repo (Q8_0 acts): GPU-CPU %.2e, CPU-double %.2e, avx2-scalar %.2e (all / sum|w x|)\n", nb, worst, ok ? "ok" : "FAIL", w_gc, w_cr, w_ac);
+ fails += !ok || w_gc > 5e-6;
+ cudaFree(mwd); if (kf.tql) { cudaFree(wd); }
}
cudaFree(dw); cudaFree(dout);
}
@@ -250,3 +278,9 @@ int main(int argc, char** argv) {
std::printf("\n%s (%d failures)\n", fails ? "FAILED" : "ALL PARITY OK", fails);
return fails ? 1 : 0;
}
+
+// tq_cpu.cpp needs these probes from expert_layout.cpp: answer from the CPU itself
+namespace strata::kernels::cpu {
+bool cpu_avx2_ok() { return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma") && __builtin_cpu_supports("f16c"); }
+bool cpu_avx512_ok() { return false; }
+}