Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

dsv41: gfx12 WMMA prompt tiles for the i-quant expert sets (STRATA_DS4_HIPK=1, default off)

Niko122194413c70570d
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cu
index 40a6ff9bf..b116133d8 100644
--- a/src/dsv41/ds4_iq.cu
+++ b/src/dsv41/ds4_iq.cu
@@ -18,6 +18,8 @@
#include <atomic>
#include <cstdio>
#include <cstdlib>
+#include <cstring>
+#include <type_traits>
#include <map>
#include <mutex>
#include <unordered_map>
@@ -1172,8 +1174,12 @@ void dn_ne(const Args& a, int groups, int ne, cudaStream_t s) {
else if (ne <= 8) launch_dn<T, 8>(a, groups, s);
else launch_dn<T, 16>(a, groups, s);
}
+#include "ds4_iq_hip.inc"
template <int T, bool GU>
void launch_set(const SetArgs& a, dim3 grid, cudaStream_t s) {
+#if defined(STRATA_USE_HIP)
+ if (hipk::launch<T, GU>(a, s)) return; // (gfx12, STRATA_DS4_HIPK=1: the matrix-core tiles)
+#endif
#if !defined(STRATA_USE_HIP)
if (tc_ok()) {
set_tc_kernel<T, GU><<<grid, kThreads, 0, s>>>(a);
diff --git a/src/dsv41/ds4_iq_hip.inc b/src/dsv41/ds4_iq_hip.inc
new file mode 100644
index 000000000..6fd2fcb4c
--- /dev/null
+++ b/src/dsv41/ds4_iq_hip.inc
@@ -0,0 +1,274 @@
+// Strata. The prompt sets of ds4_iq.cu on the gfx12 (RDNA4, R9700 = gfx1201) matrix cores: v_wmma_f32_16x16x16_f16, F16 operands,
+// F32 accumulation. Included by ds4_iq.cu inside its anonymous namespace (it uses its Ty, decode<>, load_tab<>, swiglu, SetArgs).
+//
+// The idea is the CUDA prompt tiles' (set_h16ws_kernel on f-dsv41-cuk-int2): a block-tile of the expert's rows x weight rows, K in
+// steps of 64; each i-quant block is decoded ONCE into shared memory as F16 (code * the sub-block's scale: the product is exact up
+// to the F16 rounding of the scale and the product) and then shared by every row of the tile; x as F16 rows next to it, both
+// double-buffered, one barrier a step. The F32 FMA tile of today (set_kernel) decodes the weights once per 64 rows and does
+// scalar FMAs. Default OFF: STRATA_DS4_HIPK=1 switches it on (gfx12 only, the types below, experts of more than 16 rows).
+#pragma once
+#if defined(STRATA_USE_HIP)
+
+namespace hipk {
+constexpr int kStH = 72; // halves a shared row: 64 + 8 (the pad spreads the banks; rows stay 16-byte aligned)
+using hv2 = _Float16 __attribute__((ext_vector_type(2)));
+using hv8 = _Float16 __attribute__((ext_vector_type(8)));
+using fv8 = float __attribute__((ext_vector_type(8)));
+
+template <int T>
+constexpr bool kTypes = T == kIQ2_XXS || T == kIQ2_XS || T == kIQ2_S || T == kIQ3_XXS || T == kIQ3_S || T == kIQ4_XS ||
+ T == kIQ1_S || T == kIQ1_M || T == kIQ4_NL;
+
+__device__ __forceinline__ fv8 wmma16(hv8 a, hv8 b, fv8 c) {
+#if defined(__gfx1200__) || defined(__gfx1201__)
+ return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a, b, c);
+#else
+ return c;
+#endif
+}
+__device__ __forceinline__ uint32_t as_u32(hv2 v) { return __builtin_bit_cast(uint32_t, v); }
+__device__ __forceinline__ hv2 as_hv2(uint32_t v) { return __builtin_bit_cast(hv2, v); }
+// 4 int8 codes (a word) -> 4 halves times the scale: 1024 + (code + 128) is exact in a half's mantissa, less 1152
+__device__ __forceinline__ uint2 codes4(int w, hv2 sc) {
+ const uint32_t u = (uint32_t) w ^ 0x80808080u;
+ const uint32_t lo = __byte_perm(u, 0x64646464u, 0x5140), hi = __byte_perm(u, 0x64646464u, 0x5342);
+ const hv2 m = {(_Float16) 1152.0f, (_Float16) 1152.0f};
+ const hv2 a = (as_hv2(lo) - m) * sc, b = (as_hv2(hi) - m) * sc;
+ return make_uint2(as_u32(a), as_u32(b));
+}
+
+// BM rows x BN weight rows (a matrix: gate and up: two) per block, 8 waves WM x WN, each wave (BM/WM) x (BN/WN)
+template <int T, bool GU, int BM, int BN, int WM, int WN>
+__global__ void __launch_bounds__(256) set_kernel(const SetArgs a, int dbg) {
+ static_assert(WM * WN == 8, "8 waves");
+ constexpr int NM = GU ? 2 : 1, MT = BM / WM / 16, NT = BN / WN / 16;
+ static_assert(MT >= 1 && NT >= 1 && MT * WM * 16 == BM && NT * WN * 16 == BN, "wave tiling");
+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+ __shared__ int srow[BM];
+ __shared__ __align__(16) _Float16 xs[2][BM][kStH];
+ __shared__ __align__(16) _Float16 ws[2][NM][BN][kStH];
+ const int e = blockIdx.z;
+ const int lo = a.bounds[e] - a.row0, hi_r = a.bounds[e + 1] - a.row0;
+ const int r0 = lo + blockIdx.x * BM;
+ if (r0 >= hi_r || hi_r - lo <= a.skip_le) return; // (uniform)
+ load_tab<T>(tab);
+ const int nr = min(BM, hi_r - r0);
+ const int c0 = blockIdx.y * BN;
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ const int K = GU ? a.n_embd : a.n_ff;
+ const size_t rb = row_bytes_t<T>(K);
+ const int tid = (int) threadIdx.x, warp = tid >> 5, lane = tid & 31, l16 = lane & 15, hi = lane >> 4;
+ if (tid < BM) srow[tid] = GU ? a.idx[r0 + min(tid, nr - 1)] : r0 + min(tid, nr - 1);
+ __syncthreads();
+ const int nst = K / 64;
+
+ // a step's operands are fetched and decoded into registers (the global loads and the codebook's lookups) while the matrix
+ // cores work on the step before, and written to shared memory after them
+ constexpr int NI = (NM * BN * 2 + 255) / 256; // weight items a thread
+ constexpr int NX = GU ? (BM * 16 + 255) / 256 : (BM * 8 + 255) / 256; // row chunks a thread
+ struct Regs {
+ int q[NI][8];
+ _Float16 sc0[NI], sc1[NI];
+ float4 xv[GU ? NX : 1];
+ uint4 xu[GU ? 1 : NX];
+ };
+ auto fetch = [&](int s, Regs& R) {
+#pragma unroll
+ for (int ii = 0; ii < NI; ++ii) {
+ const int it = tid + ii * 256;
+ if (it < NM * BN * 2) {
+ const int kbl = it & 1, col = (it >> 1) % BN, mat = it / (2 * BN);
+ Sub sb;
+ if (dbg & 2) { for (int w = 0; w < 8; ++w) sb.q[w] = tid + w; sb.d = 1.0f; sb.s0 = sb.s1 = 1; } else
+ decode<T>(blob + a.tr[GU ? mat : 2] + (size_t) (c0 + col) * rb, s * 2 + kbl, tab, sb);
+#pragma unroll
+ for (int w = 0; w < 8; ++w) R.q[ii][w] = sb.q[w];
+ R.sc0[ii] = (_Float16) (sb.d * (float) sb.s0);
+ R.sc1[ii] = (_Float16) (sb.d * (float) sb.s1);
+ }
+ }
+#pragma unroll
+ for (int xi = 0; xi < NX; ++xi) {
+ const int c = tid + xi * 256;
+ if constexpr (GU) {
+ if (c < BM * 16) {
+ const int row = c >> 4, ch = c & 15;
+ R.xv[xi] = *(const float4*) (a.xf + (size_t) srow[row] * a.ldx + (size_t) s * 64 + ch * 4);
+ }
+ } else {
+ if (c < BM * 8) {
+ const int row = c >> 3, ch = c & 7;
+ R.xu[xi] = *(const uint4*) (a.xh + (size_t) srow[row] * a.ldx + (size_t) s * 64 + ch * 8);
+ }
+ }
+ }
+ };
+ auto store = [&](const Regs& R, int b) {
+#pragma unroll
+ for (int ii = 0; ii < NI; ++ii) {
+ const int it = tid + ii * 256;
+ if (it < NM * BN * 2) {
+ const int kbl = it & 1, col = (it >> 1) % BN, mat = it / (2 * BN);
+ const hv2 h0 = {R.sc0[ii], R.sc0[ii]}, h1 = {R.sc1[ii], R.sc1[ii]};
+ uint2* dst = (uint2*) &ws[b][mat][col][kbl * 32]; // (values 4 w .. 4 w + 3 of the half at dst[w], 16 + 4 w .. at dst[4 + w])
+#pragma unroll
+ for (int w = 0; w < 4; ++w) {
+ dst[w] = codes4(R.q[ii][w], h0);
+ dst[4 + w] = codes4(R.q[ii][4 + w], h1);
+ }
+ }
+ }
+#pragma unroll
+ for (int xi = 0; xi < NX; ++xi) {
+ const int c = tid + xi * 256;
+ if constexpr (GU) {
+ if (c < BM * 16) {
+ const int row = c >> 4, ch = c & 15;
+ const float4 v = R.xv[xi];
+ const hv2 p = {(_Float16) v.x, (_Float16) v.y}, q = {(_Float16) v.z, (_Float16) v.w};
+ *(uint2*) &xs[b][row][ch * 4] = make_uint2(as_u32(p), as_u32(q));
+ }
+ } else {
+ if (c < BM * 8) {
+ const int row = c >> 3, ch = c & 7;
+ *(uint4*) &xs[b][row][ch * 8] = R.xu[xi];
+ }
+ }
+ }
+ };
+
+ const int wm = warp / WN, wn = warp % WN;
+ fv8 acc[NM][MT][NT];
+#pragma unroll
+ for (int m = 0; m < NM; ++m)
+#pragma unroll
+ for (int i = 0; i < MT; ++i)
+#pragma unroll
+ for (int j = 0; j < NT; ++j) acc[m][i][j] = fv8{0, 0, 0, 0, 0, 0, 0, 0};
+
+ // read-ahead: the first and last byte of the weights' block and a word of the rows' line of a step PF steps on, summed into a
+ // value that is never used (the loads only warm the caches; their latency is not waited for until the end)
+ int touched = 0;
+ auto touch = [&](int s) {
+ if (dbg & 4) return;
+#pragma unroll
+ for (int ii = 0; ii < NI; ++ii) {
+ const int it = tid + ii * 256;
+ if (it < NM * BN * 2) {
+ const int col = (it >> 1) % BN, mat = it / (2 * BN);
+ const uint8_t* p = blob + a.tr[GU ? mat : 2] + (size_t) (c0 + col) * rb + (size_t) ((s * 2) >> 3) * Ty<T>::B;
+ touched += p[0] + p[Ty<T>::B - 1];
+ }
+ }
+ if constexpr (GU) {
+ for (int c = tid; c < BM * 2; c += 256) touched += (int) a.xf[(size_t) srow[c >> 1] * a.ldx + (size_t) s * 64 + (c & 1) * 32];
+ } else {
+ for (int c = tid; c < BM * 2; c += 256) touched += (int) a.xh[(size_t) srow[c >> 1] * a.ldx + (size_t) s * 64 + (c & 1) * 32];
+ }
+ };
+ constexpr int PF = 4;
+ for (int s = 1; s <= PF && s < nst; ++s) touch(s + 1 < nst ? s + 1 : s);
+ {
+ Regs R0;
+ fetch(0, R0);
+ store(R0, 0);
+ }
+ __syncthreads();
+ for (int s = 0; s < nst; ++s) {
+ const int b = s & 1;
+ Regs R;
+ if (s + 1 < nst) fetch(s + 1, R);
+ if (s + 1 + PF < nst) touch(s + 1 + PF);
+#pragma unroll
+ for (int kk = 0; kk < 4; ++kk) {
+ hv8 af[MT];
+#pragma unroll
+ for (int i = 0; i < MT; ++i) af[i] = *(const hv8*) &xs[b][wm * (16 * MT) + i * 16 + l16][kk * 16 + hi * 8];
+#pragma unroll
+ for (int m = 0; m < NM; ++m)
+#pragma unroll
+ for (int j = 0; j < NT; ++j) {
+ const hv8 bf = *(const hv8*) &ws[b][m][wn * (16 * NT) + j * 16 + l16][kk * 16 + hi * 8];
+#pragma unroll
+ for (int i = 0; i < MT; ++i)
+ if (!(dbg & 1) && wm * (16 * MT) + i * 16 < nr) acc[m][i][j] = wmma16(af[i], bf, acc[m][i][j]); // (a tile past the rows: skipped, uniform)
+ }
+ }
+ if (s + 1 < nst) store(R, b ^ 1);
+ __syncthreads();
+ }
+ if (touched == 0x7fffffff && tid == 12345) a.y[0] = 0.0f;
+ // D[row = 8 hi + i][col = l16] of every tile
+#pragma unroll
+ for (int i = 0; i < MT; ++i)
+#pragma unroll
+ for (int q = 0; q < 8; ++q) {
+ const int row = wm * (16 * MT) + i * 16 + 8 * hi + q;
+ if (row >= nr) continue;
+ const size_t lr = (size_t) (r0 + row);
+#pragma unroll
+ for (int j = 0; j < NT; ++j) {
+ const int col = c0 + wn * (16 * NT) + j * 16 + l16;
+ if (GU) a.h[lr * a.ldo + col] = __half_as_ushort(__float2half_rn(swiglu(acc[0][i][j][q], acc[1][i][j][q], a.limit)));
+ else a.y[lr * a.ldo + col] = acc[0][i][j][q];
+ }
+ }
+}
+
+#include "ds4_iq_hip_r.inc"
+
+inline bool ok_device() { // gfx12 and STRATA_DS4_HIPK=1
+ static std::atomic<int> state[16];
+ int dev = 0;
+ cudaGetDevice(&dev);
+ if (dev < 0 || dev >= 16) return false;
+ int v = state[dev].load(std::memory_order_relaxed);
+ if (v == 0) {
+ cudaDeviceProp prop{};
+ const bool gfx12 = cudaGetDeviceProperties(&prop, dev) == cudaSuccess && std::strncmp(prop.gcnArchName, "gfx12", 5) == 0;
+ const char* e = std::getenv("STRATA_DS4_HIPK");
+ v = gfx12 && e != nullptr && e[0] != '0' ? 1 : 2;
+ state[dev].store(v, std::memory_order_relaxed);
+ }
+ return v == 1;
+}
+
+template <int T, bool GU, int BM, int BN, int WM, int WN>
+inline bool run(const SetArgs& a, cudaStream_t s) {
+ const int N = GU ? a.n_ff : a.n_embd;
+ const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (N / BN), (unsigned) a.n_exp);
+ static const int dbg = std::getenv("HIPK_DBG") ? std::atoi(std::getenv("HIPK_DBG")) : 0;
+ set_kernel<T, GU, BM, BN, WM, WN><<<grid, 256, 0, s>>>(a, dbg);
+ return cudaPeekAtLastError() == cudaSuccess;
+}
+// false: not taken (the caller's tiles do it)
+template <int T, bool GU>
+inline bool launch(const SetArgs& a, cudaStream_t s) {
+ if constexpr (!kTypes<T>) {
+ return false;
+ } else {
+ if (!ok_device() || a.max_rows <= 16) return false;
+ const int K = GU ? a.n_embd : a.n_ff, N = GU ? a.n_ff : a.n_embd;
+ if (K % 256 != 0 || N % 64 != 0 || a.ldx % 8 != 0 || a.ldo % 2 != 0) return false;
+ if (((uintptr_t) (GU ? (const void*) a.xf : (const void*) a.xh) & 15) != 0) return false;
+ static const int var = std::getenv("STRATA_DS4_HIPK") ? std::atoi(std::getenv("STRATA_DS4_HIPK")) : 0;
+ if (var != 2) { // 2: the shared-memory weights kernel
+ static const int cfg = std::getenv("HIPK_CFG") ? std::atoi(std::getenv("HIPK_CFG")) : 0;
+ if (cfg == 1 && a.max_rows <= 32) return runr<T, GU, 32, 4, 1>(a, s);
+ if (cfg == 2 && a.max_rows <= 32) return runr<T, GU, 32, 8, 1>(a, s);
+ if (cfg == 3 && a.max_rows <= 32) return runr<T, GU, 32, 2, 2>(a, s);
+ if (cfg == 4 && a.max_rows <= 32) return runr<T, GU, 32, 8, 2>(a, s);
+ if (cfg == 5 && a.max_rows <= 32) return runr<T, GU, 32, 2, 1>(a, s);
+ if (a.max_rows <= 32) return runr<T, GU, 32, 4, 2>(a, s);
+ if (a.max_rows <= 64) return runr<T, GU, 64, 4, 2>(a, s);
+ return runr<T, GU, 128, 8, 1>(a, s);
+ }
+ if (a.max_rows <= 32) return run<T, GU, 32, 64, 2, 4>(a, s);
+ if (a.max_rows <= 64) return run<T, GU, 64, 64, 2, 4>(a, s);
+ if constexpr (GU) return run<T, GU, 128, 32, 4, 2>(a, s);
+ else return run<T, GU, 128, 64, 4, 2>(a, s);
+ }
+}
+} // namespace hipk
+
+#endif
diff --git a/src/dsv41/ds4_iq_hip_r.inc b/src/dsv41/ds4_iq_hip_r.inc
new file mode 100644
index 000000000..736606605
--- /dev/null
+++ b/src/dsv41/ds4_iq_hip_r.inc
@@ -0,0 +1,177 @@
+// Strata. Part of ds4_iq_hip.inc (included from it): the weights straight into the matrix cores' registers.
+// One lane of the B operand of a 16x16x16 step holds 8 consecutive k of one weight row: exactly one 8-value group of a 32-value
+// sub-block of an i-quant block (the grid entry, its signs). grp_dec decodes that group alone: 8 int8 codes (two words) and the
+// scale (weight = scale * code), so a wave needs no shared memory for the weights and no barrier: every wave decodes its own
+// columns once and uses them for every row of the tile. kb: the 32-value sub-block of the row, p: its group (0..3).
+// The decode is in two halves so that the loads of a k-step can be issued one k-step ahead of its use: grp_load fetches the
+// group raw bytes into registers (Raw), grp_dec turns them into codes and scale (the codebook lookups, the signs).
+__device__ __forceinline__ float hf(uint32_t h) { return __half2float(__ushort_as_half((uint16_t) h)); }
+#include "ds4_iq_hip_s.inc"
+
+constexpr int kKC = 64; // K a step: two sub-blocks
+// BM rows (MT tiles of 16) x BN = NW 16 NTW weight rows per block; NW waves, each NTW column tiles of 16 and every row tile
+template <int T, bool GU, int BM, int NW, int NTW>
+__global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg) {
+ constexpr int NM = GU ? 2 : 1, MT = BM / 16, BN = NW * 16 * NTW, NTH = NW * 32;
+ static_assert(BM % 16 == 0, "row tiles");
+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];
+ __shared__ int srow[BM];
+ __shared__ __align__(16) _Float16 xs[2][BM][kStH];
+ const int e = blockIdx.z;
+ const int lo = a.bounds[e] - a.row0, hi_r = a.bounds[e + 1] - a.row0;
+ const int r0 = lo + blockIdx.x * BM;
+ if (r0 >= hi_r || hi_r - lo <= a.skip_le) return; // (uniform)
+ load_tab<T>(tab);
+ const int nr = min(BM, hi_r - r0);
+ const int c0 = blockIdx.y * BN;
+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;
+ const int K = GU ? a.n_embd : a.n_ff;
+ const size_t rb = row_bytes_t<T>(K);
+ const int tid = (int) threadIdx.x, warp = tid >> 5, lane = tid & 31, l16 = lane & 15, hi = lane >> 4;
+ for (int i = tid; i < BM; i += NTH) srow[i] = GU ? a.idx[r0 + min(i, nr - 1)] : r0 + min(i, nr - 1);
+ __syncthreads();
+ const int nst = K / kKC;
+ // the weight rows of this lane column tiles
+ const uint8_t* wrow[NM][NTW];
+#pragma unroll
+ for (int m = 0; m < NM; ++m)
+#pragma unroll
+ for (int j = 0; j < NTW; ++j)
+ wrow[m][j] = blob + a.tr[GU ? m : 2] + (size_t) ((c0 + warp * (16 * NTW) + j * 16 + l16)) * rb;
+
+ constexpr int NX = GU ? (BM * 16 + NTH - 1) / NTH : (BM * 8 + NTH - 1) / NTH;
+ struct XR {
+ float4 f[GU ? NX : 1];
+ uint4 u[GU ? 1 : NX];
+ };
+ auto fetchx = [&](int s, XR& R) {
+#pragma unroll
+ for (int xi = 0; xi < NX; ++xi) {
+ const int c = tid + xi * NTH;
+ if constexpr (GU) {
+ if (c < BM * 16) {
+ const int row = c >> 4, ch = c & 15;
+ R.f[xi] = *(const float4*) (a.xf + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 4);
+ }
+ } else {
+ if (c < BM * 8) {
+ const int row = c >> 3, ch = c & 7;
+ R.u[xi] = *(const uint4*) (a.xh + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 8);
+ }
+ }
+ }
+ };
+ auto storex = [&](const XR& R, int b) {
+#pragma unroll
+ for (int xi = 0; xi < NX; ++xi) {
+ const int c = tid + xi * NTH;
+ if constexpr (GU) {
+ if (c < BM * 16) {
+ const int row = c >> 4, ch = c & 15;
+ const float4 v = R.f[xi];
+ const hv2 p = {(_Float16) v.x, (_Float16) v.y}, q = {(_Float16) v.z, (_Float16) v.w};
+ *(uint2*) &xs[b][row][ch * 4] = make_uint2(as_u32(p), as_u32(q));
+ }
+ } else {
+ if (c < BM * 8) {
+ const int row = c >> 3, ch = c & 7;
+ *(uint4*) &xs[b][row][ch * 8] = R.u[xi];
+ }
+ }
+ }
+ };
+
+ fv8 acc[NM][MT][NTW];
+#pragma unroll
+ for (int m = 0; m < NM; ++m)
+#pragma unroll
+ for (int i = 0; i < MT; ++i)
+#pragma unroll
+ for (int j = 0; j < NTW; ++j) acc[m][i][j] = fv8{0, 0, 0, 0, 0, 0, 0, 0};
+
+ {
+ XR R0;
+ fetchx(0, R0);
+ storex(R0, 0);
+ }
+ // the weights of a step (64 values) are loaded one step ahead: two sets of registers, the step loop unrolled by two
+ SRaw sr0[NM][NTW], sr1[NM][NTW];
+ auto load_stage = [&](SRaw (&dst)[NM][NTW], int s) {
+#pragma unroll
+ for (int m = 0; m < NM; ++m)
+#pragma unroll
+ for (int j = 0; j < NTW; ++j) stage_load<T>(wrow[m][j], s, hi, dst[m][j]);
+ };
+ auto do_stage = [&](int s, SRaw (&cur)[NM][NTW], SRaw (&nxt)[NM][NTW]) {
+ const int b = s & 1;
+ XR R;
+ if (s + 1 < nst) {
+ fetchx(s + 1, R);
+ load_stage(nxt, s + 1);
+ }
+ hv8 bf[NM][NTW];
+ auto step = [&](auto tt) {
+ constexpr int t = decltype(tt)::value;
+ constexpr int sub = t >> 1, ks = t & 1;
+#pragma unroll
+ for (int m = 0; m < NM; ++m)
+#pragma unroll
+ for (int j = 0; j < NTW; ++j) {
+ int c0_, c1_;
+ float sc;
+ step_dec<T, t>(cur[m][j], s, hi, tab, c0_, c1_, sc);
+ const hv2 h = {(_Float16) sc, (_Float16) sc};
+ const uint2 u0 = codes4(c0_, h), u1 = codes4(c1_, h);
+ bf[m][j] = __builtin_bit_cast(hv8, make_uint4(u0.x, u0.y, u1.x, u1.y));
+ }
+ const int kcol = sub * 32 + ks * 16 + hi * 8;
+#pragma unroll
+ for (int i = 0; i < MT; ++i) {
+ if (i * 16 < nr) { // (a tile past the rows: skipped, uniform)
+ const hv8 af = *(const hv8*) &xs[b][i * 16 + l16][kcol];
+#pragma unroll
+ for (int m = 0; m < NM; ++m)
+#pragma unroll
+ for (int j = 0; j < NTW; ++j) acc[m][i][j] = wmma16(af, bf[m][j], acc[m][i][j]);
+ }
+ }
+ };
+ step(std::integral_constant<int, 0>{});
+ step(std::integral_constant<int, 1>{});
+ step(std::integral_constant<int, 2>{});
+ step(std::integral_constant<int, 3>{});
+ if (s + 1 < nst) storex(R, b ^ 1);
+ __syncthreads();
+ };
+ load_stage(sr0, 0);
+ __syncthreads();
+ for (int s = 0; s < nst; s += 2) { // (nst is a multiple of 4)
+ do_stage(s, sr0, sr1);
+ do_stage(s + 1, sr1, sr0);
+ }
+#pragma unroll
+ for (int i = 0; i < MT; ++i)
+#pragma unroll
+ for (int q = 0; q < 8; ++q) {
+ const int row = i * 16 + 8 * hi + q;
+ if (row >= nr) continue;
+ const size_t lr = (size_t) (r0 + row);
+#pragma unroll
+ for (int j = 0; j < NTW; ++j) {
+ const int col = c0 + warp * (16 * NTW) + j * 16 + l16;
+ if (GU) a.h[lr * a.ldo + col] = __half_as_ushort(__float2half_rn(swiglu(acc[0][i][j][q], acc[1][i][j][q], a.limit)));
+ else a.y[lr * a.ldo + col] = acc[0][i][j][q];
+ }
+ }
+}
+
+template <int T, bool GU, int BM, int NW, int NTW>
+inline bool runr(const SetArgs& a, cudaStream_t s) {
+ const int N = GU ? a.n_ff : a.n_embd;
+ constexpr int BN = NW * 16 * NTW;
+ if (N % BN != 0) return false;
+ const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (N / BN), (unsigned) a.n_exp);
+ static const int dbg = std::getenv("HIPK_DBG") ? std::atoi(std::getenv("HIPK_DBG")) : 0;
+ set_kernel_r<T, GU, BM, NW, NTW><<<grid, NW * 32, 0, s>>>(a, dbg);
+ return cudaPeekAtLastError() == cudaSuccess;
+}
diff --git a/src/dsv41/ds4_iq_hip_s.inc b/src/dsv41/ds4_iq_hip_s.inc
new file mode 100644
index 000000000..1dee41ad0
--- /dev/null
+++ b/src/dsv41/ds4_iq_hip_s.inc
@@ -0,0 +1,162 @@
+// Strata. Part of ds4_iq_hip_r.inc: the weights of a whole 64-value step (two 32-value sub-blocks) of one weight row in a few wide
+// loads, and the decode of its four 8-value groups (k-steps) from registers. The loads are unaligned (the blocks of the i-quants are
+// 2-byte aligned; the AMD hardware takes unaligned vector loads). A lane holds the group p = 2 ks + hi of every sub-block, so
+// the two lanes of a column (hi 0 / 1) share most of their bytes. s: the step (64 values), b: the block (256 values: 4 steps),
+// jb = 2 (s & 3): the first sub-block of the step in its block.
+struct SRaw {
+ uint32_t w[10];
+};
+__device__ __forceinline__ uint4 ldu128(const void* p) {
+ uint4 v;
+ __builtin_memcpy(&v, p, 16);
+ return v;
+}
+__device__ __forceinline__ uint2 ldu64(const void* p) {
+ uint2 v;
+ __builtin_memcpy(&v, p, 8);
+ return v;
+}
+__device__ __forceinline__ uint32_t ldu32(const void* p) {
+ uint32_t v;
+ __builtin_memcpy(&v, p, 4);
+ return v;
+}
+__device__ __forceinline__ uint32_t ldu16(const void* p) {
+ uint16_t v;
+ __builtin_memcpy(&v, p, 2);
+ return v;
+}
+template <int T>
+__device__ __forceinline__ void stage_load(const uint8_t* __restrict__ row, int s, int hi, SRaw& r) {
+ if constexpr (T == kIQ4_NL) { // 32-value blocks of 18 bytes: the step is blocks 2 s and 2 s + 1
+ const uint8_t* b0 = row + (size_t) (2 * s) * 18;
+ const uint2 qa = ldu64(b0 + 2 + 8 * hi), qb = ldu64(b0 + 18 + 2 + 8 * hi);
+ r.w[0] = qa.x, r.w[1] = qa.y, r.w[2] = qb.x, r.w[3] = qb.y;
+ r.w[4] = ldu16(b0);
+ r.w[5] = ldu16(b0 + 18);
+ } else {
+ const uint8_t* b = row + (size_t) (s >> 2) * Ty<T>::B;
+ const int jb = 2 * (s & 3);
+ if constexpr (T == kIQ2_XXS) {
+ const uint4 q = ldu128(b + 2 + 8 * jb);
+ r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;
+ r.w[4] = ldu16(b);
+ } else if constexpr (T == kIQ2_XS) {
+ const uint4 q = ldu128(b + 2 + 8 * jb);
+ r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;
+ r.w[4] = ldu16(b + 66 + jb);
+ r.w[5] = ldu16(b);
+ } else if constexpr (T == kIQ2_S) {
+ const uint2 q = ldu64(b + 2 + 4 * jb), sg = ldu64(b + 34 + 4 * jb);
+ r.w[0] = q.x, r.w[1] = q.y, r.w[2] = sg.x, r.w[3] = sg.y;
+ r.w[4] = ldu16(b + 66 + jb);
+ r.w[5] = ldu16(b + 74 + jb);
+ r.w[6] = ldu16(b);
+ } else if constexpr (T == kIQ3_XXS) {
+ const uint4 q = ldu128(b + 2 + 8 * jb);
+ const uint2 ax = ldu64(b + 66 + 4 * jb);
+ r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;
+ r.w[4] = ax.x, r.w[5] = ax.y;
+ r.w[6] = ldu16(b);
+ } else if constexpr (T == kIQ3_S) {
+ const uint4 q = ldu128(b + 2 + 8 * jb);
+ const uint2 sg = ldu64(b + 74 + 4 * jb);
+ r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;
+ r.w[4] = ldu16(b + 66 + jb);
+ r.w[5] = sg.x, r.w[6] = sg.y;
+ r.w[7] = b[106 + (jb >> 1)];
+ r.w[8] = ldu16(b);
+ } else if constexpr (T == kIQ4_XS) {
+ const uint2 qa = ldu64(b + 8 + 16 * jb + 8 * hi), qb = ldu64(b + 8 + 16 * (jb + 1) + 8 * hi);
+ r.w[0] = qa.x, r.w[1] = qa.y, r.w[2] = qb.x, r.w[3] = qb.y;
+ r.w[4] = ldu16(b + 2);
+ r.w[5] = b[4 + (jb >> 1)];
+ r.w[6] = ldu16(b);
+ } else if constexpr (T == kIQ1_S) {
+ const uint2 q = ldu64(b + 2 + 4 * jb);
+ r.w[0] = q.x, r.w[1] = q.y;
+ r.w[2] = ldu32(b + 34 + 2 * jb);
+ r.w[3] = ldu16(b);
+ } else { // IQ1_M
+ const uint2 q = ldu64(b + 4 * jb);
+ const uint2 d4 = ldu64(b + 48);
+ r.w[0] = q.x, r.w[1] = q.y;
+ r.w[2] = ldu32(b + 32 + 2 * jb);
+ r.w[3] = ldu16(b + 48 + (jb >> 1) * 2);
+ r.w[4] = d4.x, r.w[5] = d4.y;
+ }
+ }
+}
+// the k-step t (0..3, compile time) of a loaded step: sub-block sub = t >> 1, group p = 2 (t & 1) + hi
+template <int T, int TT>
+__device__ __forceinline__ void step_dec(const SRaw& r, int s, int hi, const uint32_t* tab, int& c0, int& c1, float& sc) {
+ constexpr int sub = TT >> 1, ks = TT & 1;
+ const int p = 2 * ks + hi;
+ if constexpr (T == kIQ2_XXS) {
+ const uint2 g = ((const uint2*) tab)[(r.w[2 * sub] >> (8 * p)) & 255u];
+ const uint32_t a1 = r.w[2 * sub + 1];
+ const uint32_t sg = ksigns8(a1 >> (7 * p));
+ c0 = sgn4(g.x, sg);
+ c1 = sgn4(g.y, sg >> 4);
+ sc = hf(r.w[4]) * (float) (2 * (int) (a1 >> 28) + 1) * 0.125f;
+ } else if constexpr (T == kIQ2_XS) {
+ const uint32_t v = (r.w[2 * sub + (ks)] >> (16 * hi)) & 0xffffu;
+ const uint2 g = ((const uint2*) tab)[v & 511u];
+ const uint32_t sg = ksigns8(v >> 9);
+ const uint32_t scb = (r.w[4] >> (8 * sub)) & 255u;
+ c0 = sgn4(g.x, sg);
+ c1 = sgn4(g.y, sg >> 4);
+ sc = hf(r.w[5]) * (float) (2 * (int) (ks == 0 ? scb & 15 : scb >> 4) + 1) * 0.125f;
+ } else if constexpr (T == kIQ2_S) {
+ const uint32_t qh = (r.w[4] >> (8 * sub)) & 255u;
+ const uint32_t idx = ((r.w[sub] >> (8 * p)) & 255u) | ((qh << (8 - 2 * p)) & 0x300u);
+ const uint32_t sg = (r.w[2 + sub] >> (8 * p)) & 255u;
+ const uint32_t scb = (r.w[5] >> (8 * sub)) & 255u;
+ const uint2 g = ((const uint2*) tab)[idx];
+ c0 = sgn4(g.x, sg);
+ c1 = sgn4(g.y, sg >> 4);
+ sc = hf(r.w[6]) * (float) (2 * (int) (ks == 0 ? scb & 15 : scb >> 4) + 1) * 0.125f;
+ } else if constexpr (T == kIQ3_XXS) {
+ const uint32_t word = r.w[2 * sub + ks];
+ const uint32_t aux = r.w[4 + sub];
+ const uint32_t sg = ksigns8(aux >> (7 * p));
+ c0 = sgn4(tab[(word >> (16 * hi)) & 255u], sg);
+ c1 = sgn4(tab[(word >> (16 * hi + 8)) & 255u], sg >> 4);
+ sc = hf(r.w[6]) * (float) (2 * (int) (aux >> 28) + 1) * 0.25f;
+ } else if constexpr (T == kIQ3_S) {
+ const uint32_t word = r.w[2 * sub + ks];
+ const uint32_t h = (r.w[4] >> (8 * sub)) & 255u;
+ const uint32_t sg = (r.w[5 + sub] >> (8 * p)) & 255u;
+ c0 = sgn4(tab[((word >> (16 * hi)) & 255u) | ((h << (8 - 2 * p)) & 256u)], sg);
+ c1 = sgn4(tab[((word >> (16 * hi + 8)) & 255u) | ((h << (7 - 2 * p)) & 256u)], sg >> 4);
+ sc = hf(r.w[8]) * (float) (1 + 2 * (int) ((r.w[7] >> (4 * sub)) & 15));
+ } else if constexpr (T == kIQ4_XS) {
+ const int jb = 2 * (s & 3);
+ const int ls = ((r.w[5] >> (4 * sub)) & 15) | (((r.w[4] >> (2 * (jb + sub))) & 3) << 4);
+ const int2 v0 = tab16(r.w[2 * sub], tab), v1 = tab16(r.w[2 * sub + 1], tab);
+ c0 = ks == 0 ? v0.x : v0.y;
+ c1 = ks == 0 ? v1.x : v1.y;
+ sc = hf(r.w[6]) * (float) (ls - 32);
+ } else if constexpr (T == kIQ1_S) {
+ const uint32_t qh = (r.w[2] >> (16 * sub)) & 0xffffu;
+ const uint32_t g = tab[((r.w[sub] >> (8 * p)) & 255u) | (((qh >> (3 * p)) & 7u) << 8)];
+ const bool neg = (qh & 0x8000u) != 0;
+ c0 = (int) iq1_codes(g & 0x0F0F0F0Fu, neg);
+ c1 = (int) iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg);
+ sc = hf(r.w[3]) * (float) (2 * (int) ((qh >> 12) & 7u) + 1) * 0.125f;
+ } else if constexpr (T == kIQ1_M) {
+ const uint32_t qh = (r.w[2] >> (8 * (2 * sub + ks))) & 255u; // (p < 2: the first byte of the sub-block, else the second)
+ const uint32_t g = tab[((r.w[sub] >> (8 * p)) & 255u) | ((qh << (hi ? 4 : 8)) & 0x700u)];
+ const bool neg = (hi ? qh & 0x80u : qh & 0x08u) != 0;
+ c0 = (int) iq1_codes(g & 0x0F0F0F0Fu, neg);
+ c1 = (int) iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg);
+ const int sh = (int) ((r.w[3] >> (6 * sub + (ks == 0 ? 0 : 3))) & 7u);
+ const uint32_t d16 = (r.w[4] >> 12 & 0xfu) | ((r.w[4] >> 24) & 0xf0u) | ((r.w[5] >> 4) & 0xf00u) | ((r.w[5] >> 16) & 0xf000u);
+ sc = hf(d16) * (float) (2 * sh + 1) * 0.125f;
+ } else { // IQ4_NL
+ const int2 v0 = tab16(r.w[2 * sub], tab), v1 = tab16(r.w[2 * sub + 1], tab);
+ c0 = ks == 0 ? v0.x : v0.y;
+ c1 = ks == 0 ? v1.x : v1.y;
+ sc = hf(r.w[4 + sub]);
+ }
+}
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 1d724de87..2126ab33c 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -7465,8 +7465,10 @@ struct Model::Impl {
for (auto& kv : agg) tot += kv.second.ms;
for (auto& kv : agg)
std::fprintf(stderr,
- "ds4 prof expert kernels type %-8s %-7s %9.1f ms calls %5ld rows/expert %6.1f ~%5.1f TFLOPS (2048-wide ffn)\n",
+ "ds4 prof expert kernels type %-8s %-7s %9.1f ms calls %5ld experts %6ld (%.1f us each, %.1f a call) rows/expert %6.1f ~%5.1f TFLOPS\n",
gg::iq::name(kv.first.first), kv.first.second ? "down" : "gate+up", kv.second.ms, kv.second.calls,
+ kv.second.experts, kv.second.experts > 0 ? 1e3 * kv.second.ms / (double) kv.second.experts : 0.0,
+ kv.second.calls > 0 ? (double) kv.second.experts / (double) kv.second.calls : 0.0,
kv.second.experts > 0 ? (double) kv.second.rows / (double) kv.second.experts : 0.0,
kv.second.ms > 0 ? kv.second.flops / kv.second.ms / 1e9 : 0.0);
std::fprintf(stderr, "ds4 prof expert kernels total %.1f ms\n", tot);