Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41: gfx12 WMMA prompt tiles for the i-quant expert sets (STRATA_DS4_HIPK=1, default off)
Niko122194413c70570d
diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex 40a6ff9bf..b116133d8 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -18,6 +18,8 @@#include <atomic>#include <cstdio>#include <cstdlib>+#include <cstring>+#include <type_traits>#include <map>#include <mutex>#include <unordered_map>@@ -1172,8 +1174,12 @@ void dn_ne(const Args& a, int groups, int ne, cudaStream_t s) {else if (ne <= 8) launch_dn<T, 8>(a, groups, s);else launch_dn<T, 16>(a, groups, s);}+#include "ds4_iq_hip.inc"template <int T, bool GU>void launch_set(const SetArgs& a, dim3 grid, cudaStream_t s) {+#if defined(STRATA_USE_HIP)+ if (hipk::launch<T, GU>(a, s)) return; // (gfx12, STRATA_DS4_HIPK=1: the matrix-core tiles)+#endif#if !defined(STRATA_USE_HIP)if (tc_ok()) {set_tc_kernel<T, GU><<<grid, kThreads, 0, s>>>(a);diff --git a/src/dsv41/ds4_iq_hip.inc b/src/dsv41/ds4_iq_hip.incnew file mode 100644index 000000000..6fd2fcb4c--- /dev/null+++ b/src/dsv41/ds4_iq_hip.inc@@ -0,0 +1,274 @@+// Strata. The prompt sets of ds4_iq.cu on the gfx12 (RDNA4, R9700 = gfx1201) matrix cores: v_wmma_f32_16x16x16_f16, F16 operands,+// F32 accumulation. Included by ds4_iq.cu inside its anonymous namespace (it uses its Ty, decode<>, load_tab<>, swiglu, SetArgs).+//+// The idea is the CUDA prompt tiles' (set_h16ws_kernel on f-dsv41-cuk-int2): a block-tile of the expert's rows x weight rows, K in+// steps of 64; each i-quant block is decoded ONCE into shared memory as F16 (code * the sub-block's scale: the product is exact up+// to the F16 rounding of the scale and the product) and then shared by every row of the tile; x as F16 rows next to it, both+// double-buffered, one barrier a step. The F32 FMA tile of today (set_kernel) decodes the weights once per 64 rows and does+// scalar FMAs. Default OFF: STRATA_DS4_HIPK=1 switches it on (gfx12 only, the types below, experts of more than 16 rows).+#pragma once+#if defined(STRATA_USE_HIP)++namespace hipk {+constexpr int kStH = 72; // halves a shared row: 64 + 8 (the pad spreads the banks; rows stay 16-byte aligned)+using hv2 = _Float16 __attribute__((ext_vector_type(2)));+using hv8 = _Float16 __attribute__((ext_vector_type(8)));+using fv8 = float __attribute__((ext_vector_type(8)));++template <int T>+constexpr bool kTypes = T == kIQ2_XXS || T == kIQ2_XS || T == kIQ2_S || T == kIQ3_XXS || T == kIQ3_S || T == kIQ4_XS ||+ T == kIQ1_S || T == kIQ1_M || T == kIQ4_NL;++__device__ __forceinline__ fv8 wmma16(hv8 a, hv8 b, fv8 c) {+#if defined(__gfx1200__) || defined(__gfx1201__)+ return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a, b, c);+#else+ return c;+#endif+}+__device__ __forceinline__ uint32_t as_u32(hv2 v) { return __builtin_bit_cast(uint32_t, v); }+__device__ __forceinline__ hv2 as_hv2(uint32_t v) { return __builtin_bit_cast(hv2, v); }+// 4 int8 codes (a word) -> 4 halves times the scale: 1024 + (code + 128) is exact in a half's mantissa, less 1152+__device__ __forceinline__ uint2 codes4(int w, hv2 sc) {+ const uint32_t u = (uint32_t) w ^ 0x80808080u;+ const uint32_t lo = __byte_perm(u, 0x64646464u, 0x5140), hi = __byte_perm(u, 0x64646464u, 0x5342);+ const hv2 m = {(_Float16) 1152.0f, (_Float16) 1152.0f};+ const hv2 a = (as_hv2(lo) - m) * sc, b = (as_hv2(hi) - m) * sc;+ return make_uint2(as_u32(a), as_u32(b));+}++// BM rows x BN weight rows (a matrix: gate and up: two) per block, 8 waves WM x WN, each wave (BM/WM) x (BN/WN)+template <int T, bool GU, int BM, int BN, int WM, int WN>+__global__ void __launch_bounds__(256) set_kernel(const SetArgs a, int dbg) {+ static_assert(WM * WN == 8, "8 waves");+ constexpr int NM = GU ? 2 : 1, MT = BM / WM / 16, NT = BN / WN / 16;+ static_assert(MT >= 1 && NT >= 1 && MT * WM * 16 == BM && NT * WN * 16 == BN, "wave tiling");+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];+ __shared__ int srow[BM];+ __shared__ __align__(16) _Float16 xs[2][BM][kStH];+ __shared__ __align__(16) _Float16 ws[2][NM][BN][kStH];+ const int e = blockIdx.z;+ const int lo = a.bounds[e] - a.row0, hi_r = a.bounds[e + 1] - a.row0;+ const int r0 = lo + blockIdx.x * BM;+ if (r0 >= hi_r || hi_r - lo <= a.skip_le) return; // (uniform)+ load_tab<T>(tab);+ const int nr = min(BM, hi_r - r0);+ const int c0 = blockIdx.y * BN;+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ const int K = GU ? a.n_embd : a.n_ff;+ const size_t rb = row_bytes_t<T>(K);+ const int tid = (int) threadIdx.x, warp = tid >> 5, lane = tid & 31, l16 = lane & 15, hi = lane >> 4;+ if (tid < BM) srow[tid] = GU ? a.idx[r0 + min(tid, nr - 1)] : r0 + min(tid, nr - 1);+ __syncthreads();+ const int nst = K / 64;++ // a step's operands are fetched and decoded into registers (the global loads and the codebook's lookups) while the matrix+ // cores work on the step before, and written to shared memory after them+ constexpr int NI = (NM * BN * 2 + 255) / 256; // weight items a thread+ constexpr int NX = GU ? (BM * 16 + 255) / 256 : (BM * 8 + 255) / 256; // row chunks a thread+ struct Regs {+ int q[NI][8];+ _Float16 sc0[NI], sc1[NI];+ float4 xv[GU ? NX : 1];+ uint4 xu[GU ? 1 : NX];+ };+ auto fetch = [&](int s, Regs& R) {+#pragma unroll+ for (int ii = 0; ii < NI; ++ii) {+ const int it = tid + ii * 256;+ if (it < NM * BN * 2) {+ const int kbl = it & 1, col = (it >> 1) % BN, mat = it / (2 * BN);+ Sub sb;+ if (dbg & 2) { for (int w = 0; w < 8; ++w) sb.q[w] = tid + w; sb.d = 1.0f; sb.s0 = sb.s1 = 1; } else+ decode<T>(blob + a.tr[GU ? mat : 2] + (size_t) (c0 + col) * rb, s * 2 + kbl, tab, sb);+#pragma unroll+ for (int w = 0; w < 8; ++w) R.q[ii][w] = sb.q[w];+ R.sc0[ii] = (_Float16) (sb.d * (float) sb.s0);+ R.sc1[ii] = (_Float16) (sb.d * (float) sb.s1);+ }+ }+#pragma unroll+ for (int xi = 0; xi < NX; ++xi) {+ const int c = tid + xi * 256;+ if constexpr (GU) {+ if (c < BM * 16) {+ const int row = c >> 4, ch = c & 15;+ R.xv[xi] = *(const float4*) (a.xf + (size_t) srow[row] * a.ldx + (size_t) s * 64 + ch * 4);+ }+ } else {+ if (c < BM * 8) {+ const int row = c >> 3, ch = c & 7;+ R.xu[xi] = *(const uint4*) (a.xh + (size_t) srow[row] * a.ldx + (size_t) s * 64 + ch * 8);+ }+ }+ }+ };+ auto store = [&](const Regs& R, int b) {+#pragma unroll+ for (int ii = 0; ii < NI; ++ii) {+ const int it = tid + ii * 256;+ if (it < NM * BN * 2) {+ const int kbl = it & 1, col = (it >> 1) % BN, mat = it / (2 * BN);+ const hv2 h0 = {R.sc0[ii], R.sc0[ii]}, h1 = {R.sc1[ii], R.sc1[ii]};+ uint2* dst = (uint2*) &ws[b][mat][col][kbl * 32]; // (values 4 w .. 4 w + 3 of the half at dst[w], 16 + 4 w .. at dst[4 + w])+#pragma unroll+ for (int w = 0; w < 4; ++w) {+ dst[w] = codes4(R.q[ii][w], h0);+ dst[4 + w] = codes4(R.q[ii][4 + w], h1);+ }+ }+ }+#pragma unroll+ for (int xi = 0; xi < NX; ++xi) {+ const int c = tid + xi * 256;+ if constexpr (GU) {+ if (c < BM * 16) {+ const int row = c >> 4, ch = c & 15;+ const float4 v = R.xv[xi];+ const hv2 p = {(_Float16) v.x, (_Float16) v.y}, q = {(_Float16) v.z, (_Float16) v.w};+ *(uint2*) &xs[b][row][ch * 4] = make_uint2(as_u32(p), as_u32(q));+ }+ } else {+ if (c < BM * 8) {+ const int row = c >> 3, ch = c & 7;+ *(uint4*) &xs[b][row][ch * 8] = R.xu[xi];+ }+ }+ }+ };++ const int wm = warp / WN, wn = warp % WN;+ fv8 acc[NM][MT][NT];+#pragma unroll+ for (int m = 0; m < NM; ++m)+#pragma unroll+ for (int i = 0; i < MT; ++i)+#pragma unroll+ for (int j = 0; j < NT; ++j) acc[m][i][j] = fv8{0, 0, 0, 0, 0, 0, 0, 0};++ // read-ahead: the first and last byte of the weights' block and a word of the rows' line of a step PF steps on, summed into a+ // value that is never used (the loads only warm the caches; their latency is not waited for until the end)+ int touched = 0;+ auto touch = [&](int s) {+ if (dbg & 4) return;+#pragma unroll+ for (int ii = 0; ii < NI; ++ii) {+ const int it = tid + ii * 256;+ if (it < NM * BN * 2) {+ const int col = (it >> 1) % BN, mat = it / (2 * BN);+ const uint8_t* p = blob + a.tr[GU ? mat : 2] + (size_t) (c0 + col) * rb + (size_t) ((s * 2) >> 3) * Ty<T>::B;+ touched += p[0] + p[Ty<T>::B - 1];+ }+ }+ if constexpr (GU) {+ for (int c = tid; c < BM * 2; c += 256) touched += (int) a.xf[(size_t) srow[c >> 1] * a.ldx + (size_t) s * 64 + (c & 1) * 32];+ } else {+ for (int c = tid; c < BM * 2; c += 256) touched += (int) a.xh[(size_t) srow[c >> 1] * a.ldx + (size_t) s * 64 + (c & 1) * 32];+ }+ };+ constexpr int PF = 4;+ for (int s = 1; s <= PF && s < nst; ++s) touch(s + 1 < nst ? s + 1 : s);+ {+ Regs R0;+ fetch(0, R0);+ store(R0, 0);+ }+ __syncthreads();+ for (int s = 0; s < nst; ++s) {+ const int b = s & 1;+ Regs R;+ if (s + 1 < nst) fetch(s + 1, R);+ if (s + 1 + PF < nst) touch(s + 1 + PF);+#pragma unroll+ for (int kk = 0; kk < 4; ++kk) {+ hv8 af[MT];+#pragma unroll+ for (int i = 0; i < MT; ++i) af[i] = *(const hv8*) &xs[b][wm * (16 * MT) + i * 16 + l16][kk * 16 + hi * 8];+#pragma unroll+ for (int m = 0; m < NM; ++m)+#pragma unroll+ for (int j = 0; j < NT; ++j) {+ const hv8 bf = *(const hv8*) &ws[b][m][wn * (16 * NT) + j * 16 + l16][kk * 16 + hi * 8];+#pragma unroll+ for (int i = 0; i < MT; ++i)+ if (!(dbg & 1) && wm * (16 * MT) + i * 16 < nr) acc[m][i][j] = wmma16(af[i], bf, acc[m][i][j]); // (a tile past the rows: skipped, uniform)+ }+ }+ if (s + 1 < nst) store(R, b ^ 1);+ __syncthreads();+ }+ if (touched == 0x7fffffff && tid == 12345) a.y[0] = 0.0f;+ // D[row = 8 hi + i][col = l16] of every tile+#pragma unroll+ for (int i = 0; i < MT; ++i)+#pragma unroll+ for (int q = 0; q < 8; ++q) {+ const int row = wm * (16 * MT) + i * 16 + 8 * hi + q;+ if (row >= nr) continue;+ const size_t lr = (size_t) (r0 + row);+#pragma unroll+ for (int j = 0; j < NT; ++j) {+ const int col = c0 + wn * (16 * NT) + j * 16 + l16;+ if (GU) a.h[lr * a.ldo + col] = __half_as_ushort(__float2half_rn(swiglu(acc[0][i][j][q], acc[1][i][j][q], a.limit)));+ else a.y[lr * a.ldo + col] = acc[0][i][j][q];+ }+ }+}++#include "ds4_iq_hip_r.inc"++inline bool ok_device() { // gfx12 and STRATA_DS4_HIPK=1+ static std::atomic<int> state[16];+ int dev = 0;+ cudaGetDevice(&dev);+ if (dev < 0 || dev >= 16) return false;+ int v = state[dev].load(std::memory_order_relaxed);+ if (v == 0) {+ cudaDeviceProp prop{};+ const bool gfx12 = cudaGetDeviceProperties(&prop, dev) == cudaSuccess && std::strncmp(prop.gcnArchName, "gfx12", 5) == 0;+ const char* e = std::getenv("STRATA_DS4_HIPK");+ v = gfx12 && e != nullptr && e[0] != '0' ? 1 : 2;+ state[dev].store(v, std::memory_order_relaxed);+ }+ return v == 1;+}++template <int T, bool GU, int BM, int BN, int WM, int WN>+inline bool run(const SetArgs& a, cudaStream_t s) {+ const int N = GU ? a.n_ff : a.n_embd;+ const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (N / BN), (unsigned) a.n_exp);+ static const int dbg = std::getenv("HIPK_DBG") ? std::atoi(std::getenv("HIPK_DBG")) : 0;+ set_kernel<T, GU, BM, BN, WM, WN><<<grid, 256, 0, s>>>(a, dbg);+ return cudaPeekAtLastError() == cudaSuccess;+}+// false: not taken (the caller's tiles do it)+template <int T, bool GU>+inline bool launch(const SetArgs& a, cudaStream_t s) {+ if constexpr (!kTypes<T>) {+ return false;+ } else {+ if (!ok_device() || a.max_rows <= 16) return false;+ const int K = GU ? a.n_embd : a.n_ff, N = GU ? a.n_ff : a.n_embd;+ if (K % 256 != 0 || N % 64 != 0 || a.ldx % 8 != 0 || a.ldo % 2 != 0) return false;+ if (((uintptr_t) (GU ? (const void*) a.xf : (const void*) a.xh) & 15) != 0) return false;+ static const int var = std::getenv("STRATA_DS4_HIPK") ? std::atoi(std::getenv("STRATA_DS4_HIPK")) : 0;+ if (var != 2) { // 2: the shared-memory weights kernel+ static const int cfg = std::getenv("HIPK_CFG") ? std::atoi(std::getenv("HIPK_CFG")) : 0;+ if (cfg == 1 && a.max_rows <= 32) return runr<T, GU, 32, 4, 1>(a, s);+ if (cfg == 2 && a.max_rows <= 32) return runr<T, GU, 32, 8, 1>(a, s);+ if (cfg == 3 && a.max_rows <= 32) return runr<T, GU, 32, 2, 2>(a, s);+ if (cfg == 4 && a.max_rows <= 32) return runr<T, GU, 32, 8, 2>(a, s);+ if (cfg == 5 && a.max_rows <= 32) return runr<T, GU, 32, 2, 1>(a, s);+ if (a.max_rows <= 32) return runr<T, GU, 32, 4, 2>(a, s);+ if (a.max_rows <= 64) return runr<T, GU, 64, 4, 2>(a, s);+ return runr<T, GU, 128, 8, 1>(a, s);+ }+ if (a.max_rows <= 32) return run<T, GU, 32, 64, 2, 4>(a, s);+ if (a.max_rows <= 64) return run<T, GU, 64, 64, 2, 4>(a, s);+ if constexpr (GU) return run<T, GU, 128, 32, 4, 2>(a, s);+ else return run<T, GU, 128, 64, 4, 2>(a, s);+ }+}+} // namespace hipk++#endifdiff --git a/src/dsv41/ds4_iq_hip_r.inc b/src/dsv41/ds4_iq_hip_r.incnew file mode 100644index 000000000..736606605--- /dev/null+++ b/src/dsv41/ds4_iq_hip_r.inc@@ -0,0 +1,177 @@+// Strata. Part of ds4_iq_hip.inc (included from it): the weights straight into the matrix cores' registers.+// One lane of the B operand of a 16x16x16 step holds 8 consecutive k of one weight row: exactly one 8-value group of a 32-value+// sub-block of an i-quant block (the grid entry, its signs). grp_dec decodes that group alone: 8 int8 codes (two words) and the+// scale (weight = scale * code), so a wave needs no shared memory for the weights and no barrier: every wave decodes its own+// columns once and uses them for every row of the tile. kb: the 32-value sub-block of the row, p: its group (0..3).+// The decode is in two halves so that the loads of a k-step can be issued one k-step ahead of its use: grp_load fetches the+// group raw bytes into registers (Raw), grp_dec turns them into codes and scale (the codebook lookups, the signs).+__device__ __forceinline__ float hf(uint32_t h) { return __half2float(__ushort_as_half((uint16_t) h)); }+#include "ds4_iq_hip_s.inc"++constexpr int kKC = 64; // K a step: two sub-blocks+// BM rows (MT tiles of 16) x BN = NW 16 NTW weight rows per block; NW waves, each NTW column tiles of 16 and every row tile+template <int T, bool GU, int BM, int NW, int NTW>+__global__ void __launch_bounds__(NW * 32) set_kernel_r(const SetArgs a, int dbg) {+ constexpr int NM = GU ? 2 : 1, MT = BM / 16, BN = NW * 16 * NTW, NTH = NW * 32;+ static_assert(BM % 16 == 0, "row tiles");+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];+ __shared__ int srow[BM];+ __shared__ __align__(16) _Float16 xs[2][BM][kStH];+ const int e = blockIdx.z;+ const int lo = a.bounds[e] - a.row0, hi_r = a.bounds[e + 1] - a.row0;+ const int r0 = lo + blockIdx.x * BM;+ if (r0 >= hi_r || hi_r - lo <= a.skip_le) return; // (uniform)+ load_tab<T>(tab);+ const int nr = min(BM, hi_r - r0);+ const int c0 = blockIdx.y * BN;+ const uint8_t* blob = a.wbase + (size_t) e * a.stride;+ const int K = GU ? a.n_embd : a.n_ff;+ const size_t rb = row_bytes_t<T>(K);+ const int tid = (int) threadIdx.x, warp = tid >> 5, lane = tid & 31, l16 = lane & 15, hi = lane >> 4;+ for (int i = tid; i < BM; i += NTH) srow[i] = GU ? a.idx[r0 + min(i, nr - 1)] : r0 + min(i, nr - 1);+ __syncthreads();+ const int nst = K / kKC;+ // the weight rows of this lane column tiles+ const uint8_t* wrow[NM][NTW];+#pragma unroll+ for (int m = 0; m < NM; ++m)+#pragma unroll+ for (int j = 0; j < NTW; ++j)+ wrow[m][j] = blob + a.tr[GU ? m : 2] + (size_t) ((c0 + warp * (16 * NTW) + j * 16 + l16)) * rb;++ constexpr int NX = GU ? (BM * 16 + NTH - 1) / NTH : (BM * 8 + NTH - 1) / NTH;+ struct XR {+ float4 f[GU ? NX : 1];+ uint4 u[GU ? 1 : NX];+ };+ auto fetchx = [&](int s, XR& R) {+#pragma unroll+ for (int xi = 0; xi < NX; ++xi) {+ const int c = tid + xi * NTH;+ if constexpr (GU) {+ if (c < BM * 16) {+ const int row = c >> 4, ch = c & 15;+ R.f[xi] = *(const float4*) (a.xf + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 4);+ }+ } else {+ if (c < BM * 8) {+ const int row = c >> 3, ch = c & 7;+ R.u[xi] = *(const uint4*) (a.xh + (size_t) srow[row] * a.ldx + (size_t) s * kKC + ch * 8);+ }+ }+ }+ };+ auto storex = [&](const XR& R, int b) {+#pragma unroll+ for (int xi = 0; xi < NX; ++xi) {+ const int c = tid + xi * NTH;+ if constexpr (GU) {+ if (c < BM * 16) {+ const int row = c >> 4, ch = c & 15;+ const float4 v = R.f[xi];+ const hv2 p = {(_Float16) v.x, (_Float16) v.y}, q = {(_Float16) v.z, (_Float16) v.w};+ *(uint2*) &xs[b][row][ch * 4] = make_uint2(as_u32(p), as_u32(q));+ }+ } else {+ if (c < BM * 8) {+ const int row = c >> 3, ch = c & 7;+ *(uint4*) &xs[b][row][ch * 8] = R.u[xi];+ }+ }+ }+ };++ fv8 acc[NM][MT][NTW];+#pragma unroll+ for (int m = 0; m < NM; ++m)+#pragma unroll+ for (int i = 0; i < MT; ++i)+#pragma unroll+ for (int j = 0; j < NTW; ++j) acc[m][i][j] = fv8{0, 0, 0, 0, 0, 0, 0, 0};++ {+ XR R0;+ fetchx(0, R0);+ storex(R0, 0);+ }+ // the weights of a step (64 values) are loaded one step ahead: two sets of registers, the step loop unrolled by two+ SRaw sr0[NM][NTW], sr1[NM][NTW];+ auto load_stage = [&](SRaw (&dst)[NM][NTW], int s) {+#pragma unroll+ for (int m = 0; m < NM; ++m)+#pragma unroll+ for (int j = 0; j < NTW; ++j) stage_load<T>(wrow[m][j], s, hi, dst[m][j]);+ };+ auto do_stage = [&](int s, SRaw (&cur)[NM][NTW], SRaw (&nxt)[NM][NTW]) {+ const int b = s & 1;+ XR R;+ if (s + 1 < nst) {+ fetchx(s + 1, R);+ load_stage(nxt, s + 1);+ }+ hv8 bf[NM][NTW];+ auto step = [&](auto tt) {+ constexpr int t = decltype(tt)::value;+ constexpr int sub = t >> 1, ks = t & 1;+#pragma unroll+ for (int m = 0; m < NM; ++m)+#pragma unroll+ for (int j = 0; j < NTW; ++j) {+ int c0_, c1_;+ float sc;+ step_dec<T, t>(cur[m][j], s, hi, tab, c0_, c1_, sc);+ const hv2 h = {(_Float16) sc, (_Float16) sc};+ const uint2 u0 = codes4(c0_, h), u1 = codes4(c1_, h);+ bf[m][j] = __builtin_bit_cast(hv8, make_uint4(u0.x, u0.y, u1.x, u1.y));+ }+ const int kcol = sub * 32 + ks * 16 + hi * 8;+#pragma unroll+ for (int i = 0; i < MT; ++i) {+ if (i * 16 < nr) { // (a tile past the rows: skipped, uniform)+ const hv8 af = *(const hv8*) &xs[b][i * 16 + l16][kcol];+#pragma unroll+ for (int m = 0; m < NM; ++m)+#pragma unroll+ for (int j = 0; j < NTW; ++j) acc[m][i][j] = wmma16(af, bf[m][j], acc[m][i][j]);+ }+ }+ };+ step(std::integral_constant<int, 0>{});+ step(std::integral_constant<int, 1>{});+ step(std::integral_constant<int, 2>{});+ step(std::integral_constant<int, 3>{});+ if (s + 1 < nst) storex(R, b ^ 1);+ __syncthreads();+ };+ load_stage(sr0, 0);+ __syncthreads();+ for (int s = 0; s < nst; s += 2) { // (nst is a multiple of 4)+ do_stage(s, sr0, sr1);+ do_stage(s + 1, sr1, sr0);+ }+#pragma unroll+ for (int i = 0; i < MT; ++i)+#pragma unroll+ for (int q = 0; q < 8; ++q) {+ const int row = i * 16 + 8 * hi + q;+ if (row >= nr) continue;+ const size_t lr = (size_t) (r0 + row);+#pragma unroll+ for (int j = 0; j < NTW; ++j) {+ const int col = c0 + warp * (16 * NTW) + j * 16 + l16;+ if (GU) a.h[lr * a.ldo + col] = __half_as_ushort(__float2half_rn(swiglu(acc[0][i][j][q], acc[1][i][j][q], a.limit)));+ else a.y[lr * a.ldo + col] = acc[0][i][j][q];+ }+ }+}++template <int T, bool GU, int BM, int NW, int NTW>+inline bool runr(const SetArgs& a, cudaStream_t s) {+ const int N = GU ? a.n_ff : a.n_embd;+ constexpr int BN = NW * 16 * NTW;+ if (N % BN != 0) return false;+ const dim3 grid((unsigned) ((a.max_rows + BM - 1) / BM), (unsigned) (N / BN), (unsigned) a.n_exp);+ static const int dbg = std::getenv("HIPK_DBG") ? std::atoi(std::getenv("HIPK_DBG")) : 0;+ set_kernel_r<T, GU, BM, NW, NTW><<<grid, NW * 32, 0, s>>>(a, dbg);+ return cudaPeekAtLastError() == cudaSuccess;+}diff --git a/src/dsv41/ds4_iq_hip_s.inc b/src/dsv41/ds4_iq_hip_s.incnew file mode 100644index 000000000..1dee41ad0--- /dev/null+++ b/src/dsv41/ds4_iq_hip_s.inc@@ -0,0 +1,162 @@+// Strata. Part of ds4_iq_hip_r.inc: the weights of a whole 64-value step (two 32-value sub-blocks) of one weight row in a few wide+// loads, and the decode of its four 8-value groups (k-steps) from registers. The loads are unaligned (the blocks of the i-quants are+// 2-byte aligned; the AMD hardware takes unaligned vector loads). A lane holds the group p = 2 ks + hi of every sub-block, so+// the two lanes of a column (hi 0 / 1) share most of their bytes. s: the step (64 values), b: the block (256 values: 4 steps),+// jb = 2 (s & 3): the first sub-block of the step in its block.+struct SRaw {+ uint32_t w[10];+};+__device__ __forceinline__ uint4 ldu128(const void* p) {+ uint4 v;+ __builtin_memcpy(&v, p, 16);+ return v;+}+__device__ __forceinline__ uint2 ldu64(const void* p) {+ uint2 v;+ __builtin_memcpy(&v, p, 8);+ return v;+}+__device__ __forceinline__ uint32_t ldu32(const void* p) {+ uint32_t v;+ __builtin_memcpy(&v, p, 4);+ return v;+}+__device__ __forceinline__ uint32_t ldu16(const void* p) {+ uint16_t v;+ __builtin_memcpy(&v, p, 2);+ return v;+}+template <int T>+__device__ __forceinline__ void stage_load(const uint8_t* __restrict__ row, int s, int hi, SRaw& r) {+ if constexpr (T == kIQ4_NL) { // 32-value blocks of 18 bytes: the step is blocks 2 s and 2 s + 1+ const uint8_t* b0 = row + (size_t) (2 * s) * 18;+ const uint2 qa = ldu64(b0 + 2 + 8 * hi), qb = ldu64(b0 + 18 + 2 + 8 * hi);+ r.w[0] = qa.x, r.w[1] = qa.y, r.w[2] = qb.x, r.w[3] = qb.y;+ r.w[4] = ldu16(b0);+ r.w[5] = ldu16(b0 + 18);+ } else {+ const uint8_t* b = row + (size_t) (s >> 2) * Ty<T>::B;+ const int jb = 2 * (s & 3);+ if constexpr (T == kIQ2_XXS) {+ const uint4 q = ldu128(b + 2 + 8 * jb);+ r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;+ r.w[4] = ldu16(b);+ } else if constexpr (T == kIQ2_XS) {+ const uint4 q = ldu128(b + 2 + 8 * jb);+ r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;+ r.w[4] = ldu16(b + 66 + jb);+ r.w[5] = ldu16(b);+ } else if constexpr (T == kIQ2_S) {+ const uint2 q = ldu64(b + 2 + 4 * jb), sg = ldu64(b + 34 + 4 * jb);+ r.w[0] = q.x, r.w[1] = q.y, r.w[2] = sg.x, r.w[3] = sg.y;+ r.w[4] = ldu16(b + 66 + jb);+ r.w[5] = ldu16(b + 74 + jb);+ r.w[6] = ldu16(b);+ } else if constexpr (T == kIQ3_XXS) {+ const uint4 q = ldu128(b + 2 + 8 * jb);+ const uint2 ax = ldu64(b + 66 + 4 * jb);+ r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;+ r.w[4] = ax.x, r.w[5] = ax.y;+ r.w[6] = ldu16(b);+ } else if constexpr (T == kIQ3_S) {+ const uint4 q = ldu128(b + 2 + 8 * jb);+ const uint2 sg = ldu64(b + 74 + 4 * jb);+ r.w[0] = q.x, r.w[1] = q.y, r.w[2] = q.z, r.w[3] = q.w;+ r.w[4] = ldu16(b + 66 + jb);+ r.w[5] = sg.x, r.w[6] = sg.y;+ r.w[7] = b[106 + (jb >> 1)];+ r.w[8] = ldu16(b);+ } else if constexpr (T == kIQ4_XS) {+ const uint2 qa = ldu64(b + 8 + 16 * jb + 8 * hi), qb = ldu64(b + 8 + 16 * (jb + 1) + 8 * hi);+ r.w[0] = qa.x, r.w[1] = qa.y, r.w[2] = qb.x, r.w[3] = qb.y;+ r.w[4] = ldu16(b + 2);+ r.w[5] = b[4 + (jb >> 1)];+ r.w[6] = ldu16(b);+ } else if constexpr (T == kIQ1_S) {+ const uint2 q = ldu64(b + 2 + 4 * jb);+ r.w[0] = q.x, r.w[1] = q.y;+ r.w[2] = ldu32(b + 34 + 2 * jb);+ r.w[3] = ldu16(b);+ } else { // IQ1_M+ const uint2 q = ldu64(b + 4 * jb);+ const uint2 d4 = ldu64(b + 48);+ r.w[0] = q.x, r.w[1] = q.y;+ r.w[2] = ldu32(b + 32 + 2 * jb);+ r.w[3] = ldu16(b + 48 + (jb >> 1) * 2);+ r.w[4] = d4.x, r.w[5] = d4.y;+ }+ }+}+// the k-step t (0..3, compile time) of a loaded step: sub-block sub = t >> 1, group p = 2 (t & 1) + hi+template <int T, int TT>+__device__ __forceinline__ void step_dec(const SRaw& r, int s, int hi, const uint32_t* tab, int& c0, int& c1, float& sc) {+ constexpr int sub = TT >> 1, ks = TT & 1;+ const int p = 2 * ks + hi;+ if constexpr (T == kIQ2_XXS) {+ const uint2 g = ((const uint2*) tab)[(r.w[2 * sub] >> (8 * p)) & 255u];+ const uint32_t a1 = r.w[2 * sub + 1];+ const uint32_t sg = ksigns8(a1 >> (7 * p));+ c0 = sgn4(g.x, sg);+ c1 = sgn4(g.y, sg >> 4);+ sc = hf(r.w[4]) * (float) (2 * (int) (a1 >> 28) + 1) * 0.125f;+ } else if constexpr (T == kIQ2_XS) {+ const uint32_t v = (r.w[2 * sub + (ks)] >> (16 * hi)) & 0xffffu;+ const uint2 g = ((const uint2*) tab)[v & 511u];+ const uint32_t sg = ksigns8(v >> 9);+ const uint32_t scb = (r.w[4] >> (8 * sub)) & 255u;+ c0 = sgn4(g.x, sg);+ c1 = sgn4(g.y, sg >> 4);+ sc = hf(r.w[5]) * (float) (2 * (int) (ks == 0 ? scb & 15 : scb >> 4) + 1) * 0.125f;+ } else if constexpr (T == kIQ2_S) {+ const uint32_t qh = (r.w[4] >> (8 * sub)) & 255u;+ const uint32_t idx = ((r.w[sub] >> (8 * p)) & 255u) | ((qh << (8 - 2 * p)) & 0x300u);+ const uint32_t sg = (r.w[2 + sub] >> (8 * p)) & 255u;+ const uint32_t scb = (r.w[5] >> (8 * sub)) & 255u;+ const uint2 g = ((const uint2*) tab)[idx];+ c0 = sgn4(g.x, sg);+ c1 = sgn4(g.y, sg >> 4);+ sc = hf(r.w[6]) * (float) (2 * (int) (ks == 0 ? scb & 15 : scb >> 4) + 1) * 0.125f;+ } else if constexpr (T == kIQ3_XXS) {+ const uint32_t word = r.w[2 * sub + ks];+ const uint32_t aux = r.w[4 + sub];+ const uint32_t sg = ksigns8(aux >> (7 * p));+ c0 = sgn4(tab[(word >> (16 * hi)) & 255u], sg);+ c1 = sgn4(tab[(word >> (16 * hi + 8)) & 255u], sg >> 4);+ sc = hf(r.w[6]) * (float) (2 * (int) (aux >> 28) + 1) * 0.25f;+ } else if constexpr (T == kIQ3_S) {+ const uint32_t word = r.w[2 * sub + ks];+ const uint32_t h = (r.w[4] >> (8 * sub)) & 255u;+ const uint32_t sg = (r.w[5 + sub] >> (8 * p)) & 255u;+ c0 = sgn4(tab[((word >> (16 * hi)) & 255u) | ((h << (8 - 2 * p)) & 256u)], sg);+ c1 = sgn4(tab[((word >> (16 * hi + 8)) & 255u) | ((h << (7 - 2 * p)) & 256u)], sg >> 4);+ sc = hf(r.w[8]) * (float) (1 + 2 * (int) ((r.w[7] >> (4 * sub)) & 15));+ } else if constexpr (T == kIQ4_XS) {+ const int jb = 2 * (s & 3);+ const int ls = ((r.w[5] >> (4 * sub)) & 15) | (((r.w[4] >> (2 * (jb + sub))) & 3) << 4);+ const int2 v0 = tab16(r.w[2 * sub], tab), v1 = tab16(r.w[2 * sub + 1], tab);+ c0 = ks == 0 ? v0.x : v0.y;+ c1 = ks == 0 ? v1.x : v1.y;+ sc = hf(r.w[6]) * (float) (ls - 32);+ } else if constexpr (T == kIQ1_S) {+ const uint32_t qh = (r.w[2] >> (16 * sub)) & 0xffffu;+ const uint32_t g = tab[((r.w[sub] >> (8 * p)) & 255u) | (((qh >> (3 * p)) & 7u) << 8)];+ const bool neg = (qh & 0x8000u) != 0;+ c0 = (int) iq1_codes(g & 0x0F0F0F0Fu, neg);+ c1 = (int) iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg);+ sc = hf(r.w[3]) * (float) (2 * (int) ((qh >> 12) & 7u) + 1) * 0.125f;+ } else if constexpr (T == kIQ1_M) {+ const uint32_t qh = (r.w[2] >> (8 * (2 * sub + ks))) & 255u; // (p < 2: the first byte of the sub-block, else the second)+ const uint32_t g = tab[((r.w[sub] >> (8 * p)) & 255u) | ((qh << (hi ? 4 : 8)) & 0x700u)];+ const bool neg = (hi ? qh & 0x80u : qh & 0x08u) != 0;+ c0 = (int) iq1_codes(g & 0x0F0F0F0Fu, neg);+ c1 = (int) iq1_codes((g >> 4) & 0x0F0F0F0Fu, neg);+ const int sh = (int) ((r.w[3] >> (6 * sub + (ks == 0 ? 0 : 3))) & 7u);+ const uint32_t d16 = (r.w[4] >> 12 & 0xfu) | ((r.w[4] >> 24) & 0xf0u) | ((r.w[5] >> 4) & 0xf00u) | ((r.w[5] >> 16) & 0xf000u);+ sc = hf(d16) * (float) (2 * sh + 1) * 0.125f;+ } else { // IQ4_NL+ const int2 v0 = tab16(r.w[2 * sub], tab), v1 = tab16(r.w[2 * sub + 1], tab);+ c0 = ks == 0 ? v0.x : v0.y;+ c1 = ks == 0 ? v1.x : v1.y;+ sc = hf(r.w[4 + sub]);+ }+}diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 1d724de87..2126ab33c 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -7465,8 +7465,10 @@ struct Model::Impl {for (auto& kv : agg) tot += kv.second.ms;for (auto& kv : agg)std::fprintf(stderr,- "ds4 prof expert kernels type %-8s %-7s %9.1f ms calls %5ld rows/expert %6.1f ~%5.1f TFLOPS (2048-wide ffn)\n",+ "ds4 prof expert kernels type %-8s %-7s %9.1f ms calls %5ld experts %6ld (%.1f us each, %.1f a call) rows/expert %6.1f ~%5.1f TFLOPS\n",gg::iq::name(kv.first.first), kv.first.second ? "down" : "gate+up", kv.second.ms, kv.second.calls,+ kv.second.experts, kv.second.experts > 0 ? 1e3 * kv.second.ms / (double) kv.second.experts : 0.0,+ kv.second.calls > 0 ? (double) kv.second.experts / (double) kv.second.calls : 0.0,kv.second.experts > 0 ? (double) kv.second.rows / (double) kv.second.experts : 0.0,kv.second.ms > 0 ? kv.second.flops / kv.second.ms / 1e9 : 0.0);std::fprintf(stderr, "ds4 prof expert kernels total %.1f ms\n", tot);