Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: gfx12 WMMA trellis GEMV (ds4_tqw.cuh, from the CUDA lab's recipe) with a parity + bench test: all K4-K7, widths 1/3/5, table and computed codes within 8e-8 of the double reference on the R9700; ~1.07 Tw/s

Niko12213d8d1079bcbd
diff --git a/src/dsv41/ds4_tqw.cuh b/src/dsv41/ds4_tqw.cuh
new file mode 100644
index 000000000..ea272493f
--- /dev/null
+++ b/src/dsv41/ds4_tqw.cuh
@@ -0,0 +1,138 @@
+// Strata. gfx12 (RDNA4, R9700 = gfx1201) WMMA form of the trellis expert GEMV of the dsv41 CUDA lab (tools/dsv41_lab:
+// tqk_gemv3.cuh, wmma_emu.hpp = the specification, hip/tqk_gemv_wmma.hip.cpp = its first HIP form). Layout "TQW" (see
+// wmma_emu.hpp::repack_tqw): a wave = T tiles of 16 rows, lane l = r + 16 kg owns row r and 16 consecutive steps of every
+// 128-weight block; activations are the B operand (NB <= 16 columns), split x = hi + lo in two fp16 for ~2^-22 accuracy.
+// CODE 0: the table codebook (K5/K6/K7 of the shipped files: two lookups into an 8 KB LDS table of half2 per step);
+// CODE 1: the computed "3single" code (no table; the new K4/K5 types if they take it).
+#pragma once
+#include <hip/hip_runtime.h>
+#include <cstdint>
+
+namespace strata::ds4::tqw {
+
+typedef _Float16 half8_t __attribute__((ext_vector_type(8)));
+typedef float float8_t __attribute__((ext_vector_type(8)));
+
+constexpr uint32_t kMask = 0x7fffu;
+
+struct Job {
+ const uint32_t* words; // TQW words [tile][P][K][32 lanes]
+ const uint16_t* scales; // [tile][P][16 rows][2 blocks] fp16
+ const half8_t* xb; // prepared B fragments (prep_b)
+ float* y; // [NB][ldy]
+ int rows, in, ldy;
+};
+
+__device__ __forceinline__ void step_words(const uint32_t* lut, uint32_t s, uint32_t& w0, uint32_t& w1, int code) {
+ if (code == 1) {
+ w0 = ((s * 89226354u + 64248484u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
+ w1 = ((s * 0x9E3779B1u + 0x7F4A7C15u) & 0x8FFF8FFFu) ^ 0x3B603B60u;
+ } else {
+ const uint32_t x = s * 0x9e3779b1u;
+ w0 = lut[x >> 21];
+ w1 = lut[(x >> 10) & 2047u];
+ }
+}
+
+// activations -> B fragments. out[(((blk * 8 + m) * 2 + kg) * nb + n) * 2 + {0 = hi, 1 = lo}] = 8 halves of x[n][blk * 128 + 4 * (16 kg + 2 m) + j]
+__global__ void prep_b(const float* x, int ldx, int nb, int in, half8_t* out) {
+ const int nblk = in / 128;
+ const size_t gid = (size_t) blockIdx.x * blockDim.x + threadIdx.x, n_all = (size_t) nblk * 8 * 2 * nb;
+ if (gid >= n_all) return;
+ const int n = (int) (gid % nb), kg = (int) ((gid / nb) % 2), m = (int) ((gid / (2 * nb)) % 8), blk = (int) (gid / (16 * nb));
+ const float* p = x + (size_t) n * ldx + blk * 128 + 4 * (16 * kg + 2 * m);
+ half8_t hi, lo;
+#pragma unroll
+ for (int j = 0; j < 8; ++j) { hi[j] = (_Float16) p[j]; lo[j] = (_Float16) (p[j] - (float) hi[j]); }
+ out[gid * 2] = hi;
+ out[gid * 2 + 1] = lo;
+}
+
+template <int K, int NB, int T, int EXACT, int CODE>
+__global__ void __launch_bounds__(128) gemv(const Job* jobs, const uint32_t* __restrict__ lut_g) {
+ __shared__ uint32_t lut[CODE == 0 ? 2048 : 1];
+ if (CODE == 0) {
+ for (int i = threadIdx.x; i < 2048; i += blockDim.x) lut[i] = lut_g[i];
+ __syncthreads();
+ }
+ const Job job = jobs[blockIdx.y];
+ const int wave = threadIdx.x >> 5, lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4;
+ const int tile0 = (blockIdx.x * (blockDim.x >> 5) + wave) * T;
+ if (tile0 * 16 >= job.rows) return;
+ const int nblk = job.in / 128, NP = nblk / 2;
+ float tot[T][8];
+#pragma unroll
+ for (int t = 0; t < T; ++t)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) tot[t][i] = 0.f;
+ for (int P = 0; P < NP; ++P) {
+ uint32_t rec[T][K];
+#pragma unroll
+ for (int t = 0; t < T; ++t)
+#pragma unroll
+ for (int j = 0; j < K; ++j) rec[t][j] = __builtin_nontemporal_load(job.words + (((size_t) (tile0 + t) * NP + P) * K + j) * 32 + lane);
+#pragma unroll
+ for (int bk = 0; bk < 2; ++bk) {
+ half8_t bh[8], bl[8];
+ if (r < NB) {
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ const half8_t* p = job.xb + (((((size_t) (2 * P + bk) * 8 + m) * 2 + kg) * NB + r) * 2);
+ bh[m] = p[0];
+ if (EXACT) bl[m] = p[1];
+ }
+ } else {
+#pragma unroll
+ for (int m = 0; m < 8; ++m) { bh[m] = half8_t{0, 0, 0, 0, 0, 0, 0, 0}; bl[m] = bh[m]; }
+ }
+#pragma unroll
+ for (int t = 0; t < T; ++t) {
+ // chunk bk of the record: bits [16 K bk, 16 K bk + 16 K); window 0 = steps u 0..7 (+ 15-K bits of the partner chunk),
+ // window 1 = steps u 8..15
+ constexpr int NW = K; // (words of a record)
+ auto bits = [&](int pos, int n) -> uint64_t { // n <= 56 bits at record bit pos (pos is a compile-time constant here)
+ const int a = pos >> 5, sh = pos & 31;
+ const uint32_t w0 = rec[t][a], w1 = a + 1 < NW ? rec[t][a + 1 < NW ? a + 1 : 0] : 0u, w2 = a + 2 < NW ? rec[t][a + 2 < NW ? a + 2 : 0] : 0u;
+ // (w2:w1:w0) >> sh, low 64 bits
+ const uint64_t out = (sh == 0) ? (((uint64_t) w1 << 32) | w0) : ((((uint64_t) w1 << 32) | w0) >> sh) | ((uint64_t) w2 << (64 - sh));
+ return out & ((1ull << n) - 1ull);
+ };
+ const int b0 = 16 * K * bk;
+ const uint64_t lo8 = bits(b0, 8 * K), hi8 = bits(b0 + 8 * K, 8 * K);
+ const uint32_t mine = (uint32_t) (lo8 & 0xffffffffu); // the bits the PARTNER needs: this lane's chunk bits [0, 15 - K)
+ const uint32_t pc = __shfl_xor((int) mine, 16);
+ const uint64_t win0 = hi8 | ((uint64_t) (pc & ((1u << (15 - K)) - 1u)) << (8 * K));
+ const uint64_t win1 = lo8 | ((hi8 & ((1ull << (15 - K)) - 1ull)) << (8 * K));
+ float8_t acc = {0, 0, 0, 0, 0, 0, 0, 0};
+#pragma unroll
+ for (int m = 0; m < 8; ++m) {
+ half8_t a;
+#pragma unroll
+ for (int s2 = 0; s2 < 2; ++s2) {
+ const int u = 2 * m + s2;
+ const uint32_t st = (uint32_t) (u < 8 ? win0 >> ((7 - u) * K) : win1 >> ((15 - u) * K)) & kMask;
+ uint32_t w0, w1;
+ step_words(lut, st, w0, w1, CODE);
+ a[4 * s2 + 0] = __builtin_bit_cast(_Float16, (uint16_t) (w0 & 0xffffu));
+ a[4 * s2 + 1] = __builtin_bit_cast(_Float16, (uint16_t) (w0 >> 16));
+ a[4 * s2 + 2] = __builtin_bit_cast(_Float16, (uint16_t) (w1 & 0xffffu));
+ a[4 * s2 + 3] = __builtin_bit_cast(_Float16, (uint16_t) (w1 >> 16));
+ }
+ acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a, bh[m], acc);
+ if (EXACT) acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a, bl[m], acc);
+ }
+ // scales of rows 8 kg .. 8 kg + 7 of this block: 8 consecutive (row, bk) pairs of 2 -> stride 2
+ const uint16_t* sp = job.scales + (((size_t) (tile0 + t) * NP + P) * 16 + 8 * kg) * 2 + bk;
+#pragma unroll
+ for (int i = 0; i < 8; ++i) tot[t][i] += (float) __builtin_bit_cast(_Float16, sp[2 * i]) * acc[i];
+ }
+ }
+ }
+#pragma unroll
+ for (int t = 0; t < T; ++t)
+ if (r < NB)
+#pragma unroll
+ for (int i = 0; i < 8; ++i) job.y[(size_t) r * job.ldy + (tile0 + t) * 16 + 8 * kg + i] = tot[t][i];
+}
+
+} // namespace strata::ds4::tqw
diff --git a/tests/dsv41/tqw_hip_test.cpp b/tests/dsv41/tqw_hip_test.cpp
new file mode 100644
index 000000000..ee3b633ec
--- /dev/null
+++ b/tests/dsv41/tqw_hip_test.cpp
@@ -0,0 +1,137 @@
+// Strata. Parity + speed of the gfx12 WMMA trellis GEMV (src/dsv41/ds4_tqw.cuh) on a R9700 (gfx1201):
+// tqw_hip_test [--k 4,5,6,7] [--code 0|1] [--bench]
+// Parity: random legal streams (any bit pattern is one), host-decoded weights (tqenc::decode_blocks: the table codebook, or the
+// computed code when TQE_CODE=3single, with --code 1) against the kernel's y, relative to sum|w x| per output.
+// --bench: the DeepSeek decode shape (12 gate/up matrices of 2304 x 5120), GB/s of weight bytes.
+#include "../../src/dsv41/ds4_tqw.cuh"
+#include "../../tools/dsv41_lab/wmma_emu.hpp"
+#include "../../tools/trellis/tqenc.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <random>
+#include <string>
+#include <vector>
+
+using namespace strata::ds4::tqw;
+#define CK(x) do { hipError_t e_ = (x); if (e_ != hipSuccess) { std::printf("HIP error %s at %d\n", hipGetErrorString(e_), __LINE__); std::exit(2); } } while (0)
+
+template <typename T> static T* up(const std::vector<T>& v) { T* d; CK(hipMalloc(&d, v.size() * sizeof(T))); CK(hipMemcpy(d, v.data(), v.size() * sizeof(T), hipMemcpyHostToDevice)); return d; }
+
+template <int K, int NB, int CODE> static void launch(const Job* jd, const uint32_t* lut, int rows, int njobs) {
+ constexpr int T = 2;
+ dim3 grid((rows + 16 * T * 4 - 1) / (16 * T * 4), njobs);
+ gemv<K, NB, T, 1, CODE><<<grid, 128>>>(jd, lut);
+}
+template <int K, int CODE> static void launch_nb(int nb, const Job* jd, const uint32_t* lut, int rows, int njobs) {
+ switch (nb) {
+ case 1: launch<K, 1, CODE>(jd, lut, rows, njobs); break;
+ case 3: launch<K, 3, CODE>(jd, lut, rows, njobs); break;
+ default: launch<K, 5, CODE>(jd, lut, rows, njobs); break;
+ }
+}
+template <int CODE> static void launch_k(int K, int nb, const Job* jd, const uint32_t* lut, int rows, int njobs) {
+ switch (K) {
+ case 4: launch_nb<4, CODE>(nb, jd, lut, rows, njobs); break;
+ case 5: launch_nb<5, CODE>(nb, jd, lut, rows, njobs); break;
+ case 6: launch_nb<6, CODE>(nb, jd, lut, rows, njobs); break;
+ default: launch_nb<7, CODE>(nb, jd, lut, rows, njobs); break;
+ }
+}
+static void run(int code, int K, int nb, const Job* jd, const uint32_t* lut, int rows, int njobs) {
+ if (code) launch_k<1>(K, nb, jd, lut, rows, njobs); else launch_k<0>(K, nb, jd, lut, rows, njobs);
+}
+
+int main(int argc, char** argv) {
+ int code = std::getenv("TQE_CODE") ? 1 : 0;
+ bool bench = false;
+ std::vector<int> Ks = {4, 5, 6, 7};
+ for (int i = 1; i < argc; ++i) {
+ std::string a = argv[i];
+ if (a == "--bench") bench = true;
+ else if (a == "--code" && i + 1 < argc) code = std::atoi(argv[++i]);
+ else if (a == "--k" && i + 1 < argc) { Ks.clear(); for (const char* p = argv[++i]; *p;) { Ks.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }
+ }
+ std::vector<uint32_t> hl(2048);
+ {
+ const float* pt = tqenc::point_table();
+ for (int i = 0; i < 2048; ++i) hl[i] = tqw::f2h_rn(pt[2 * i]) | ((uint32_t) tqw::f2h_rn(pt[2 * i + 1]) << 16);
+ }
+ const uint32_t* dlut = up(hl);
+ std::mt19937_64 rng(7);
+ int fails = 0;
+ const int types[8] = {0, 0, 0, 0, 225, 224, 145, 146};
+ for (int K : Ks) {
+ const int type = types[K], bsz = 2 + 4 * K;
+ const int rows = 96, in = 1536, nblk = in / 128;
+ std::vector<uint8_t> w((size_t) rows * nblk * bsz);
+ for (auto& b : w) b = (uint8_t) rng();
+ for (int64_t b = 0; b < (int64_t) rows * nblk; ++b) { const uint16_t d = tqw::f2h_rn(0.02f + 0.001f * (float) (rng() % 400)); std::memcpy(&w[(size_t) b * bsz], &d, 2); }
+ std::vector<float> dec((size_t) rows * in);
+ tqenc::decode_blocks(type, w.data(), (int64_t) rows * nblk, dec.data());
+ std::vector<uint32_t> words; std::vector<uint16_t> sc;
+ tqw::repack_tqw(K, w.data(), rows, nblk, words, sc);
+ for (int nb : {1, 3, 5}) {
+ std::vector<float> x((size_t) nb * in);
+ std::uniform_real_distribution<float> u(-1.f, 1.f);
+ for (auto& v : x) v = u(rng);
+ const uint32_t* dw = up(words); const uint16_t* ds = up(sc); const float* dx = up(x);
+ half8_t* xb; CK(hipMalloc(&xb, (size_t) nblk * 8 * 2 * nb * 2 * sizeof(half8_t)));
+ const size_t nall = (size_t) nblk * 8 * 2 * nb;
+ prep_b<<<(unsigned) ((nall + 127) / 128), 128>>>(dx, in, nb, in, xb);
+ float* dy; CK(hipMalloc(&dy, (size_t) nb * rows * 4)); CK(hipMemset(dy, 0, (size_t) nb * rows * 4));
+ std::vector<Job> jv = {Job{dw, ds, xb, dy, rows, in, rows}};
+ const Job* dj = up(jv);
+ run(code, K, nb, dj, dlut, rows, 1);
+ CK(hipDeviceSynchronize());
+ std::vector<float> y((size_t) nb * rows);
+ CK(hipMemcpy(y.data(), dy, y.size() * 4, hipMemcpyDeviceToHost));
+ double worst = 0;
+ for (int c = 0; c < nb; ++c)
+ for (int r = 0; r < rows; ++r) {
+ double s = 0, sa = 0;
+ for (int i = 0; i < in; ++i) { const double t = (double) dec[(size_t) r * in + i] * x[(size_t) c * in + i]; s += t; sa += std::fabs(t); }
+ worst = std::max(worst, std::fabs(y[(size_t) c * rows + r] - s) / (sa + 1e-30));
+ }
+ const bool ok = worst < 3e-7;
+ std::printf("K=%d nb=%d (%s): HIP wmma kernel vs double reference %.2e %s\n", K, nb, code ? "computed" : "table", worst, ok ? "ok" : "FAIL");
+ fails += !ok;
+ }
+ }
+ if (bench) {
+ for (int K : Ks) {
+ const int bsz = 2 + 4 * K, in = 5120, rows = 2304, nblk = in / 128;
+ std::vector<uint8_t> w((size_t) rows * nblk * bsz);
+ for (auto& b : w) b = (uint8_t) rng();
+ std::vector<uint32_t> words; std::vector<uint16_t> sc;
+ tqw::repack_tqw(K, w.data(), rows, nblk, words, sc);
+ for (int nb : {1, 5}) {
+ std::vector<float> x((size_t) nb * in, 0.5f);
+ const float* dx = up(x);
+ half8_t* xb; CK(hipMalloc(&xb, (size_t) nblk * 8 * 2 * nb * 2 * sizeof(half8_t)));
+ const size_t nall = (size_t) nblk * 8 * 2 * nb;
+ prep_b<<<(unsigned) ((nall + 127) / 128), 128>>>(dx, in, nb, in, xb);
+ float* dy; CK(hipMalloc(&dy, (size_t) nb * rows * 4));
+ std::vector<Job> jv; // 12 distinct buffers, so the L2 cannot serve a repeated matrix
+ for (int j = 0; j < 12; ++j) jv.push_back(Job{up(words), up(sc), xb, dy, rows, in, rows});
+ const Job* dj = up(jv);
+ const double bytes = 12.0 * rows * nblk * bsz;
+ for (int rep = 0; rep < 3; ++rep) run(code, K, nb, dj, dlut, rows, 12);
+ CK(hipDeviceSynchronize());
+ const auto t0 = std::chrono::steady_clock::now();
+ const int reps = 50;
+ for (int rep = 0; rep < reps; ++rep) run(code, K, nb, dj, dlut, rows, 12);
+ CK(hipDeviceSynchronize());
+ const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count() / reps;
+ std::printf("bench K=%d nb=%d %s: 12 gate/up matrices %.1f us, %.0f GB/s, %.2f Tw/s\n", K, nb, code ? "computed" : "table", us, bytes / us / 1e3,
+ 12.0 * rows * in / us / 1e6);
+ }
+ }
+ }
+ std::printf("%s\n", fails ? "FAILED" : "ALL OK");
+ return fails;
+}