Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: gfx12 WMMA trellis GEMV (ds4_tqw.cuh, from the CUDA lab's recipe) with a parity + bench test: all K4-K7, widths 1/3/5, table and computed codes within 8e-8 of the double reference on the R9700; ~1.07 Tw/s
Niko12213d8d1079bcbd
diff --git a/src/dsv41/ds4_tqw.cuh b/src/dsv41/ds4_tqw.cuhnew file mode 100644index 000000000..ea272493f--- /dev/null+++ b/src/dsv41/ds4_tqw.cuh@@ -0,0 +1,138 @@+// Strata. gfx12 (RDNA4, R9700 = gfx1201) WMMA form of the trellis expert GEMV of the dsv41 CUDA lab (tools/dsv41_lab:+// tqk_gemv3.cuh, wmma_emu.hpp = the specification, hip/tqk_gemv_wmma.hip.cpp = its first HIP form). Layout "TQW" (see+// wmma_emu.hpp::repack_tqw): a wave = T tiles of 16 rows, lane l = r + 16 kg owns row r and 16 consecutive steps of every+// 128-weight block; activations are the B operand (NB <= 16 columns), split x = hi + lo in two fp16 for ~2^-22 accuracy.+// CODE 0: the table codebook (K5/K6/K7 of the shipped files: two lookups into an 8 KB LDS table of half2 per step);+// CODE 1: the computed "3single" code (no table; the new K4/K5 types if they take it).+#pragma once+#include <hip/hip_runtime.h>+#include <cstdint>++namespace strata::ds4::tqw {++typedef _Float16 half8_t __attribute__((ext_vector_type(8)));+typedef float float8_t __attribute__((ext_vector_type(8)));++constexpr uint32_t kMask = 0x7fffu;++struct Job {+ const uint32_t* words; // TQW words [tile][P][K][32 lanes]+ const uint16_t* scales; // [tile][P][16 rows][2 blocks] fp16+ const half8_t* xb; // prepared B fragments (prep_b)+ float* y; // [NB][ldy]+ int rows, in, ldy;+};++__device__ __forceinline__ void step_words(const uint32_t* lut, uint32_t s, uint32_t& w0, uint32_t& w1, int code) {+ if (code == 1) {+ w0 = ((s * 89226354u + 64248484u) & 0x8FFF8FFFu) ^ 0x3B603B60u;+ w1 = ((s * 0x9E3779B1u + 0x7F4A7C15u) & 0x8FFF8FFFu) ^ 0x3B603B60u;+ } else {+ const uint32_t x = s * 0x9e3779b1u;+ w0 = lut[x >> 21];+ w1 = lut[(x >> 10) & 2047u];+ }+}++// activations -> B fragments. out[(((blk * 8 + m) * 2 + kg) * nb + n) * 2 + {0 = hi, 1 = lo}] = 8 halves of x[n][blk * 128 + 4 * (16 kg + 2 m) + j]+__global__ void prep_b(const float* x, int ldx, int nb, int in, half8_t* out) {+ const int nblk = in / 128;+ const size_t gid = (size_t) blockIdx.x * blockDim.x + threadIdx.x, n_all = (size_t) nblk * 8 * 2 * nb;+ if (gid >= n_all) return;+ const int n = (int) (gid % nb), kg = (int) ((gid / nb) % 2), m = (int) ((gid / (2 * nb)) % 8), blk = (int) (gid / (16 * nb));+ const float* p = x + (size_t) n * ldx + blk * 128 + 4 * (16 * kg + 2 * m);+ half8_t hi, lo;+#pragma unroll+ for (int j = 0; j < 8; ++j) { hi[j] = (_Float16) p[j]; lo[j] = (_Float16) (p[j] - (float) hi[j]); }+ out[gid * 2] = hi;+ out[gid * 2 + 1] = lo;+}++template <int K, int NB, int T, int EXACT, int CODE>+__global__ void __launch_bounds__(128) gemv(const Job* jobs, const uint32_t* __restrict__ lut_g) {+ __shared__ uint32_t lut[CODE == 0 ? 2048 : 1];+ if (CODE == 0) {+ for (int i = threadIdx.x; i < 2048; i += blockDim.x) lut[i] = lut_g[i];+ __syncthreads();+ }+ const Job job = jobs[blockIdx.y];+ const int wave = threadIdx.x >> 5, lane = threadIdx.x & 31, r = lane & 15, kg = lane >> 4;+ const int tile0 = (blockIdx.x * (blockDim.x >> 5) + wave) * T;+ if (tile0 * 16 >= job.rows) return;+ const int nblk = job.in / 128, NP = nblk / 2;+ float tot[T][8];+#pragma unroll+ for (int t = 0; t < T; ++t)+#pragma unroll+ for (int i = 0; i < 8; ++i) tot[t][i] = 0.f;+ for (int P = 0; P < NP; ++P) {+ uint32_t rec[T][K];+#pragma unroll+ for (int t = 0; t < T; ++t)+#pragma unroll+ for (int j = 0; j < K; ++j) rec[t][j] = __builtin_nontemporal_load(job.words + (((size_t) (tile0 + t) * NP + P) * K + j) * 32 + lane);+#pragma unroll+ for (int bk = 0; bk < 2; ++bk) {+ half8_t bh[8], bl[8];+ if (r < NB) {+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ const half8_t* p = job.xb + (((((size_t) (2 * P + bk) * 8 + m) * 2 + kg) * NB + r) * 2);+ bh[m] = p[0];+ if (EXACT) bl[m] = p[1];+ }+ } else {+#pragma unroll+ for (int m = 0; m < 8; ++m) { bh[m] = half8_t{0, 0, 0, 0, 0, 0, 0, 0}; bl[m] = bh[m]; }+ }+#pragma unroll+ for (int t = 0; t < T; ++t) {+ // chunk bk of the record: bits [16 K bk, 16 K bk + 16 K); window 0 = steps u 0..7 (+ 15-K bits of the partner chunk),+ // window 1 = steps u 8..15+ constexpr int NW = K; // (words of a record)+ auto bits = [&](int pos, int n) -> uint64_t { // n <= 56 bits at record bit pos (pos is a compile-time constant here)+ const int a = pos >> 5, sh = pos & 31;+ const uint32_t w0 = rec[t][a], w1 = a + 1 < NW ? rec[t][a + 1 < NW ? a + 1 : 0] : 0u, w2 = a + 2 < NW ? rec[t][a + 2 < NW ? a + 2 : 0] : 0u;+ // (w2:w1:w0) >> sh, low 64 bits+ const uint64_t out = (sh == 0) ? (((uint64_t) w1 << 32) | w0) : ((((uint64_t) w1 << 32) | w0) >> sh) | ((uint64_t) w2 << (64 - sh));+ return out & ((1ull << n) - 1ull);+ };+ const int b0 = 16 * K * bk;+ const uint64_t lo8 = bits(b0, 8 * K), hi8 = bits(b0 + 8 * K, 8 * K);+ const uint32_t mine = (uint32_t) (lo8 & 0xffffffffu); // the bits the PARTNER needs: this lane's chunk bits [0, 15 - K)+ const uint32_t pc = __shfl_xor((int) mine, 16);+ const uint64_t win0 = hi8 | ((uint64_t) (pc & ((1u << (15 - K)) - 1u)) << (8 * K));+ const uint64_t win1 = lo8 | ((hi8 & ((1ull << (15 - K)) - 1ull)) << (8 * K));+ float8_t acc = {0, 0, 0, 0, 0, 0, 0, 0};+#pragma unroll+ for (int m = 0; m < 8; ++m) {+ half8_t a;+#pragma unroll+ for (int s2 = 0; s2 < 2; ++s2) {+ const int u = 2 * m + s2;+ const uint32_t st = (uint32_t) (u < 8 ? win0 >> ((7 - u) * K) : win1 >> ((15 - u) * K)) & kMask;+ uint32_t w0, w1;+ step_words(lut, st, w0, w1, CODE);+ a[4 * s2 + 0] = __builtin_bit_cast(_Float16, (uint16_t) (w0 & 0xffffu));+ a[4 * s2 + 1] = __builtin_bit_cast(_Float16, (uint16_t) (w0 >> 16));+ a[4 * s2 + 2] = __builtin_bit_cast(_Float16, (uint16_t) (w1 & 0xffffu));+ a[4 * s2 + 3] = __builtin_bit_cast(_Float16, (uint16_t) (w1 >> 16));+ }+ acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a, bh[m], acc);+ if (EXACT) acc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a, bl[m], acc);+ }+ // scales of rows 8 kg .. 8 kg + 7 of this block: 8 consecutive (row, bk) pairs of 2 -> stride 2+ const uint16_t* sp = job.scales + (((size_t) (tile0 + t) * NP + P) * 16 + 8 * kg) * 2 + bk;+#pragma unroll+ for (int i = 0; i < 8; ++i) tot[t][i] += (float) __builtin_bit_cast(_Float16, sp[2 * i]) * acc[i];+ }+ }+ }+#pragma unroll+ for (int t = 0; t < T; ++t)+ if (r < NB)+#pragma unroll+ for (int i = 0; i < 8; ++i) job.y[(size_t) r * job.ldy + (tile0 + t) * 16 + 8 * kg + i] = tot[t][i];+}++} // namespace strata::ds4::tqwdiff --git a/tests/dsv41/tqw_hip_test.cpp b/tests/dsv41/tqw_hip_test.cppnew file mode 100644index 000000000..ee3b633ec--- /dev/null+++ b/tests/dsv41/tqw_hip_test.cpp@@ -0,0 +1,137 @@+// Strata. Parity + speed of the gfx12 WMMA trellis GEMV (src/dsv41/ds4_tqw.cuh) on a R9700 (gfx1201):+// tqw_hip_test [--k 4,5,6,7] [--code 0|1] [--bench]+// Parity: random legal streams (any bit pattern is one), host-decoded weights (tqenc::decode_blocks: the table codebook, or the+// computed code when TQE_CODE=3single, with --code 1) against the kernel's y, relative to sum|w x| per output.+// --bench: the DeepSeek decode shape (12 gate/up matrices of 2304 x 5120), GB/s of weight bytes.+#include "../../src/dsv41/ds4_tqw.cuh"+#include "../../tools/dsv41_lab/wmma_emu.hpp"+#include "../../tools/trellis/tqenc.hpp"++#include <algorithm>+#include <chrono>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <random>+#include <string>+#include <vector>++using namespace strata::ds4::tqw;+#define CK(x) do { hipError_t e_ = (x); if (e_ != hipSuccess) { std::printf("HIP error %s at %d\n", hipGetErrorString(e_), __LINE__); std::exit(2); } } while (0)++template <typename T> static T* up(const std::vector<T>& v) { T* d; CK(hipMalloc(&d, v.size() * sizeof(T))); CK(hipMemcpy(d, v.data(), v.size() * sizeof(T), hipMemcpyHostToDevice)); return d; }++template <int K, int NB, int CODE> static void launch(const Job* jd, const uint32_t* lut, int rows, int njobs) {+ constexpr int T = 2;+ dim3 grid((rows + 16 * T * 4 - 1) / (16 * T * 4), njobs);+ gemv<K, NB, T, 1, CODE><<<grid, 128>>>(jd, lut);+}+template <int K, int CODE> static void launch_nb(int nb, const Job* jd, const uint32_t* lut, int rows, int njobs) {+ switch (nb) {+ case 1: launch<K, 1, CODE>(jd, lut, rows, njobs); break;+ case 3: launch<K, 3, CODE>(jd, lut, rows, njobs); break;+ default: launch<K, 5, CODE>(jd, lut, rows, njobs); break;+ }+}+template <int CODE> static void launch_k(int K, int nb, const Job* jd, const uint32_t* lut, int rows, int njobs) {+ switch (K) {+ case 4: launch_nb<4, CODE>(nb, jd, lut, rows, njobs); break;+ case 5: launch_nb<5, CODE>(nb, jd, lut, rows, njobs); break;+ case 6: launch_nb<6, CODE>(nb, jd, lut, rows, njobs); break;+ default: launch_nb<7, CODE>(nb, jd, lut, rows, njobs); break;+ }+}+static void run(int code, int K, int nb, const Job* jd, const uint32_t* lut, int rows, int njobs) {+ if (code) launch_k<1>(K, nb, jd, lut, rows, njobs); else launch_k<0>(K, nb, jd, lut, rows, njobs);+}++int main(int argc, char** argv) {+ int code = std::getenv("TQE_CODE") ? 1 : 0;+ bool bench = false;+ std::vector<int> Ks = {4, 5, 6, 7};+ for (int i = 1; i < argc; ++i) {+ std::string a = argv[i];+ if (a == "--bench") bench = true;+ else if (a == "--code" && i + 1 < argc) code = std::atoi(argv[++i]);+ else if (a == "--k" && i + 1 < argc) { Ks.clear(); for (const char* p = argv[++i]; *p;) { Ks.push_back(std::atoi(p)); while (*p && *p != ',') ++p; if (*p) ++p; } }+ }+ std::vector<uint32_t> hl(2048);+ {+ const float* pt = tqenc::point_table();+ for (int i = 0; i < 2048; ++i) hl[i] = tqw::f2h_rn(pt[2 * i]) | ((uint32_t) tqw::f2h_rn(pt[2 * i + 1]) << 16);+ }+ const uint32_t* dlut = up(hl);+ std::mt19937_64 rng(7);+ int fails = 0;+ const int types[8] = {0, 0, 0, 0, 225, 224, 145, 146};+ for (int K : Ks) {+ const int type = types[K], bsz = 2 + 4 * K;+ const int rows = 96, in = 1536, nblk = in / 128;+ std::vector<uint8_t> w((size_t) rows * nblk * bsz);+ for (auto& b : w) b = (uint8_t) rng();+ for (int64_t b = 0; b < (int64_t) rows * nblk; ++b) { const uint16_t d = tqw::f2h_rn(0.02f + 0.001f * (float) (rng() % 400)); std::memcpy(&w[(size_t) b * bsz], &d, 2); }+ std::vector<float> dec((size_t) rows * in);+ tqenc::decode_blocks(type, w.data(), (int64_t) rows * nblk, dec.data());+ std::vector<uint32_t> words; std::vector<uint16_t> sc;+ tqw::repack_tqw(K, w.data(), rows, nblk, words, sc);+ for (int nb : {1, 3, 5}) {+ std::vector<float> x((size_t) nb * in);+ std::uniform_real_distribution<float> u(-1.f, 1.f);+ for (auto& v : x) v = u(rng);+ const uint32_t* dw = up(words); const uint16_t* ds = up(sc); const float* dx = up(x);+ half8_t* xb; CK(hipMalloc(&xb, (size_t) nblk * 8 * 2 * nb * 2 * sizeof(half8_t)));+ const size_t nall = (size_t) nblk * 8 * 2 * nb;+ prep_b<<<(unsigned) ((nall + 127) / 128), 128>>>(dx, in, nb, in, xb);+ float* dy; CK(hipMalloc(&dy, (size_t) nb * rows * 4)); CK(hipMemset(dy, 0, (size_t) nb * rows * 4));+ std::vector<Job> jv = {Job{dw, ds, xb, dy, rows, in, rows}};+ const Job* dj = up(jv);+ run(code, K, nb, dj, dlut, rows, 1);+ CK(hipDeviceSynchronize());+ std::vector<float> y((size_t) nb * rows);+ CK(hipMemcpy(y.data(), dy, y.size() * 4, hipMemcpyDeviceToHost));+ double worst = 0;+ for (int c = 0; c < nb; ++c)+ for (int r = 0; r < rows; ++r) {+ double s = 0, sa = 0;+ for (int i = 0; i < in; ++i) { const double t = (double) dec[(size_t) r * in + i] * x[(size_t) c * in + i]; s += t; sa += std::fabs(t); }+ worst = std::max(worst, std::fabs(y[(size_t) c * rows + r] - s) / (sa + 1e-30));+ }+ const bool ok = worst < 3e-7;+ std::printf("K=%d nb=%d (%s): HIP wmma kernel vs double reference %.2e %s\n", K, nb, code ? "computed" : "table", worst, ok ? "ok" : "FAIL");+ fails += !ok;+ }+ }+ if (bench) {+ for (int K : Ks) {+ const int bsz = 2 + 4 * K, in = 5120, rows = 2304, nblk = in / 128;+ std::vector<uint8_t> w((size_t) rows * nblk * bsz);+ for (auto& b : w) b = (uint8_t) rng();+ std::vector<uint32_t> words; std::vector<uint16_t> sc;+ tqw::repack_tqw(K, w.data(), rows, nblk, words, sc);+ for (int nb : {1, 5}) {+ std::vector<float> x((size_t) nb * in, 0.5f);+ const float* dx = up(x);+ half8_t* xb; CK(hipMalloc(&xb, (size_t) nblk * 8 * 2 * nb * 2 * sizeof(half8_t)));+ const size_t nall = (size_t) nblk * 8 * 2 * nb;+ prep_b<<<(unsigned) ((nall + 127) / 128), 128>>>(dx, in, nb, in, xb);+ float* dy; CK(hipMalloc(&dy, (size_t) nb * rows * 4));+ std::vector<Job> jv; // 12 distinct buffers, so the L2 cannot serve a repeated matrix+ for (int j = 0; j < 12; ++j) jv.push_back(Job{up(words), up(sc), xb, dy, rows, in, rows});+ const Job* dj = up(jv);+ const double bytes = 12.0 * rows * nblk * bsz;+ for (int rep = 0; rep < 3; ++rep) run(code, K, nb, dj, dlut, rows, 12);+ CK(hipDeviceSynchronize());+ const auto t0 = std::chrono::steady_clock::now();+ const int reps = 50;+ for (int rep = 0; rep < reps; ++rep) run(code, K, nb, dj, dlut, rows, 12);+ CK(hipDeviceSynchronize());+ const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count() / reps;+ std::printf("bench K=%d nb=%d %s: 12 gate/up matrices %.1f us, %.0f GB/s, %.2f Tw/s\n", K, nb, code ? "computed" : "table", us, bytes / us / 1e3,+ 12.0 * rows * in / us / 1e6);+ }+ }+ }+ std::printf("%s\n", fails ? "FAILED" : "ALL OK");+ return fails;+}