Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Decode: opt-in STRATA_MMVQ_IL_PF=1 hoisted-load, exact-integer interleaved GEMV kernels for Q6_K (#3779, #3712)
linxuhao8b95bffb0bf2
diff --git a/CMakeLists.txt b/CMakeLists.txtindex a5d060e2e..250a22f1f 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1132,6 +1132,10 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_include_directories(mmvq_il_parity PRIVATE ${_strata_gpu_include_directories})target_link_libraries(mmvq_il_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})add_test(NAME mmvq_il_parity COMMAND mmvq_il_parity)+ # the opt-in STRATA_MMVQ_IL_PF kernels (hoisted group loads, exact integer forms): every variant bit-checked against+ # native_mmvq for every type, shape, T and rows choice+ add_test(NAME mmvq_il_parity_pf COMMAND mmvq_il_parity --pf 0,2,3,4,5,6)+ set_tests_properties(mmvq_il_parity_pf PROPERTIES TIMEOUT 900)# the wide dequant kernels (a thread per 8 values; sm_61's default), bitwise vs the per-group ones (GPU, synthetic,# no model; --bench adds old/new timings)add_executable(dequant_wide_parity src/kernels/dequant_wide_parity.cpp)diff --git a/include/strata/kernels/native_mmvq.hpp b/include/strata/kernels/native_mmvq.hppindex ce11095cb..1d511b079 100644--- a/include/strata/kernels/native_mmvq.hpp+++ b/include/strata/kernels/native_mmvq.hpp@@ -166,5 +166,8 @@ void native_mmvq_il(int ggml_type, const void* weights, const void* x_q8_1, consint native_mmvq_il_rows_for(int cc, int ggml_type, int ncols, int n_out);// Tests and benchmarks: every native_mmvq_il call takes `rows` a warp (0: the table).void native_mmvq_il_tune(int rows);+// Tests and benchmarks: the P1 / P2 kernel variant for every native_mmvq_il call (0: today's kernel, 2-6 as+// STRATA_MMVQ_IL_PF; -1: the environment).+void native_mmvq_il_pf(int pf);} // namespace strata::kernelsdiff --git a/src/kernels/cuda/native_mmvq.cu b/src/kernels/cuda/native_mmvq.cuindex 2cc1c75a3..4867ec7f6 100644--- a/src/kernels/cuda/native_mmvq.cu+++ b/src/kernels/cuda/native_mmvq.cu@@ -650,6 +650,66 @@ __device__ __forceinline__ float q6_q8_dot_impl(int vl, int vh, const int* __res}return d * sumf;}+#if !defined(__HIPCC__) // CUDA only: the native_mmvq_il_pf_kernel family below (itself CUDA only) is the only user+// q6_q8_dot_impl with the two scales already in registers (the interleaved kernels' hoisted loads, P1): the same+// expression in the same order+__device__ __forceinline__ float q6_q8_dot_sc(int vl, int vh, const int* __restrict__ u, int sc0, int sc1, float d,+ const float* __restrict__ d8) {+ float sumf = 0.0f;+ const int scs[2] = {sc0, sc1};+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ const int sc = scs[i];+ const int vil = (vl >> (4 * i)) & 0x0f0f0f0f;+ const int vih = ((vh >> (4 * i)) << 4) & 0x30303030;+ const int vi = __vsubss4(vil | vih, 0x20202020);+ sumf += d8[i] * (STRATA_DP4A(vi, u[i], 0) * sc);+ }+ return d * sumf;+}++// P1: loads the compiler may not sink into a later branch (it moved the Q6_K scale and d loads of the hoisted group+// into the dot sections, where each one is a dependent L1 round trip); the same ld.global.nc the __restrict__ loads use+__device__ __forceinline__ int ldg_nc_u16(const void* p) { unsigned v; asm volatile("ld.global.nc.u16 %0, [%1];" : "=r"(v) : "l"(p)); return int(v); }+__device__ __forceinline__ int ldg_nc_s8(const void* p) { int v; asm volatile("ld.global.nc.s8 %0, [%1];" : "=r"(v) : "l"(p)); return v; }+__device__ __forceinline__ int ldg_nc_u8(const void* p) { unsigned v; asm volatile("ld.global.nc.u8 %0, [%1];" : "=r"(v) : "l"(p)); return int(v); }+__device__ __forceinline__ int ldg_nc_s32(const void* p) { int v; asm volatile("ld.global.nc.s32 %0, [%1];" : "=r"(v) : "l"(p)); return v; }+__device__ __forceinline__ int ldg_nc_b2(const void* p, int i32) { // load_int_b2 pinned: two u16, little-endian+ const auto* x = static_cast<const uint16_t*>(p) + 2 * i32;+ return ldg_nc_u16(x) | (ldg_nc_u16(x + 1) << 16);+}+// P1: base + k * stride as one IMAD.WIDE.U32 (an asm so the front end does not share k * stride between pointers and+// turn each into a 64-bit add pair)+__device__ __forceinline__ const void* madw(const void* base, int k, int stride) {+ const void* r;+ asm("mad.wide.u32 %0, %1, %2, %3;" : "=l"(r) : "r"(k), "r"(stride), "l"(base));+ return r;+}+// P2: (float) p for |p| < 2^22 as an add on the FMA pipe instead of the quarter-rate I2F: p + 1.5 * 2^23+// as float bits is exactly 1.5 * 2^23 + p (unit ulp), and the subtraction is exact (Sterbenz), so the value is the+// conversion's, +0 included+__device__ __forceinline__ float i2f_small(int p_plus_magic) { return __int_as_float(p_plus_magic) - 12582912.0f; }+constexpr int kI2fMagic = 0x4B400000;+// P2: __vsubss4(x, 0x20202020) for bytes of x in [0, 63] (Q6_K's codes, so nothing saturates): x | 0x80 is x + 0x80 per+// byte, minus 0x20 never borrows (0x60..0x9f), and the xor takes the 0x80 back: the same int8 bytes, 2 instructions+// instead of ptxas's 8 for the saturating subtract sm_80 lacks+__device__ __forceinline__ int sub32_bytes(int x) { return (x + 0x60606060) ^ 0x80808080; }+// q6_q8_dot_sc with P2's integer forms: the dp4a inputs are the same 32-bit values, (dp * sc) the same int, and+// i2f_small its exact float (|dp * sc| <= 4 * 32 * 127 * 128 = 2.1M < 2^22), so every FFMA sees the bits q6_q8_dot_impl's does+__device__ __forceinline__ float q6_q8_dot_fast(int vl, int vh, const int* __restrict__ u, int sc0, int sc1, float d,+ const float* __restrict__ d8) {+ float sumf = 0.0f;+ const int scs[2] = {sc0, sc1};+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ const int vil = (vl >> (4 * i)) & 0x0f0f0f0f;+ const int vih = ((vh >> (4 * i)) << 4) & 0x30303030;+ const int vi = sub32_bytes(vil | vih);+ sumf += d8[i] * i2f_small(STRATA_DP4A(vi, u[i], 0) * scs[i] + kI2fMagic);+ }+ return d * sumf;+}+#endif // !__HIPCC____device__ __forceinline__ float q6_q8_dot(const Q6KBlock* __restrict__ w,const Q81Block* __restrict__ x, int iqs) {@@ -835,6 +895,7 @@ constexpr int MAX_NCOLS = 8;// every column bitwise equal to it.struct Q5KTraits {using Block = Q5KBlock;+ [[maybe_unused]] static constexpr int TYPE = 13; // ggml typestatic constexpr int DIV = QK, T = QI / VDR, KBY = QK / Q8K, BPI = VDR * WARPS * WARP / QI;__device__ static int kqs(int tid) { return VDR * (tid % (QI / VDR)); }struct W { int vl[2], vh[2]; uint16_t aux[2]; half2 dm; int bq8_offset; };@@ -876,6 +937,7 @@ struct Q5KTraits {};struct Q4KTraits {using Block = Q4KBlock;+ [[maybe_unused]] static constexpr int TYPE = 12; // ggml typestatic constexpr int DIV = QK, T = QI / VDR, KBY = QK / Q8K, BPI = VDR * WARPS * WARP / QI;__device__ static int kqs(int tid) { return VDR * (tid % (QI / VDR)); }struct W { int v[2]; uint16_t aux[2]; half2 dm; int bq8_offset; };@@ -946,6 +1008,7 @@ struct Q20Traits {};struct Q3KTraits {using Block = Q3KBlock;+ [[maybe_unused]] static constexpr int TYPE = 11; // ggml typestatic constexpr int DIV = 256, T = 16, KBY = 8, BPI = WARPS * WARP / 16;__device__ static int kqs(int tid) { return tid % 16; }struct W { int vl, vh; float d; const uint8_t* scales; int scale_offset, bq8_offset; };@@ -972,6 +1035,7 @@ struct Q3KTraits {};struct Q6KTraits {using Block = Q6KBlock;+ [[maybe_unused]] static constexpr int TYPE = 14; // ggml typestatic constexpr int DIV = 256, T = 32, KBY = 8, BPI = WARPS * WARP / 32;__device__ static int kqs(int tid) { return tid % 32; }struct W { int vl, vh; float d; const int8_t* scales; int bq8_offset; };@@ -999,6 +1063,7 @@ struct Q6KTraits {};struct IQ4XSTraits {using Block = IQ4XSBlock;+ [[maybe_unused]] static constexpr int TYPE = 23; // ggml typestatic constexpr int DIV = 256, T = 8, KBY = 8, BPI = 4 * WARPS * WARP / 32;__device__ static int kqs(int tid) { return 4 * (tid % 8); }struct W { int2 v[4]; int ls; float dw; };@@ -1556,10 +1621,141 @@ struct IlIQ4XS {out[c] = d * sumi;}}+ // P1 (native_mmvq_il_pf_kernel): the block's bytes as loaded, the table lookup and the scale decode deferred to+ // applyr so the loads of a whole group issue before any of them is consumed; the same integers and floats as apply+ static constexpr bool HAS_PF = true;+ struct RP { const uint8_t *qs, *sl, *blk; }; // a row's blocks for this lane: its qs ints, its scales_l byte, the block+ __device__ static RP rowptrs(const F::Block* __restrict__ row, int iqs) {+ const auto* b = reinterpret_cast<const uint8_t*>(row);+ return {b + 8 + 4 * iqs, b + 4 + iqs / 8, b};+ }+ struct WR { int q[4]; int sl, sh, d; };+ // the pointers of block kbx (RP::at), and the loads of block kbx + dv from them (dv a compile-time offset, so the+ // group's loads share one pointer set and differ by an immediate)+ __device__ static RP at(const RP& rp, int kbx) {+ constexpr int S = int(sizeof(F::Block));+ return {static_cast<const uint8_t*>(madw(rp.qs, kbx, S)), static_cast<const uint8_t*>(madw(rp.sl, kbx, S)),+ static_cast<const uint8_t*>(madw(rp.blk, kbx, S))};+ }+ __device__ static WR loadr(const RP& rp, int dv) {+ constexpr int S = int(sizeof(F::Block));+ const auto* qs = reinterpret_cast<const int*>(rp.qs + dv * S);+ WR r;+#pragma unroll+ for (int j = 0; j < 4; ++j) r.q[j] = ldg_nc_s32(qs + j);+ r.sl = ldg_nc_u8(rp.sl + dv * S);+ r.sh = ldg_nc_u16(rp.blk + dv * S + 2);+ r.d = ldg_nc_u16(rp.blk + dv * S);+ return r;+ }+ // the activations: position-major (every int of the 8 at a stride of nb), so the kernel's block pointers are not used+ __device__ static int xb(int) { return 0; }+ __device__ static int xp(int) { return 0; }+ template <int NC, bool FAST, class X>+ __device__ static void applyr(const WR& rr, const X& x, const int*, const float*, int kby, int iqs, float (&out)[NC]) {+ F::W r;+#pragma unroll+ for (int j = 0; j < 4; ++j) r.v[j] = iq4_table_lookup(rr.q[j]);+ r.ls = ((rr.sl >> (iqs & 0x04)) & 0x0f) | (((rr.sh >> (iqs / 2)) & 0x03) << 4);+ r.dw = __half2float(__ushort_as_half((unsigned short) rr.d));+ if constexpr (!FAST) {+ apply<NC>(r, x, kby, iqs, out);+ } else {+ // P2: sumi (|sumi| <= 32 * 127 * 127 = 516K < 2^22) through i2f_small, then times (float)(ls - 32): the product+ // |sumi * (ls - 32)| < 2^24 is exact in fp32, so it is the float of the int apply() converts; the same d * sumi+ const int b = kby + iqs / 4;+ int u0[4][NC], u1[4][NC];+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ x.u(b, j, u0[j]);+ x.u(b, j + 4, u1[j]);+ }+ float d8[NC];+ x.scales(b, d8);+ const float fls = float(r.ls - 32);+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ int sumi = 0;+#pragma unroll+ for (int j = 0; j < 4; ++j) {+ sumi = STRATA_DP4A(r.v[j].x, u0[j][c], sumi);+ sumi = STRATA_DP4A(r.v[j].y, u1[j][c], sumi);+ }+ const float fsumi = i2f_small(sumi + kI2fMagic) * fls;+ const float d = r.dw * d8[c];+ out[c] = d * fsumi;+ }+ }+ }};struct IlQ4K {using F = Q4KTraits;static constexpr bool PM = false;+ static constexpr bool HAS_PF = true; // P1: F::W holds every byte of the block already (no pointer into it)+ struct RP { const uint8_t *qs, *sc, *blk; int iqs; }; // this lane's ql ints, its scale words, the block (dm)+ __device__ static RP rowptrs(const F::Block* __restrict__ row, int iqs) {+ const auto* b = reinterpret_cast<const uint8_t*>(row);+ const int bq8_offset = 2 * ((iqs / 2) / 4), jm = (bq8_offset / 2) & 1;+ return {b + 16 + 16 * bq8_offset + 4 * ((iqs / 2) % 4), b + 4 + 2 * jm, b, iqs};+ }+ using WR = F::W;+ __device__ static RP at(const RP& rp, int kbx) {+ constexpr int S = int(sizeof(F::Block));+ return {static_cast<const uint8_t*>(madw(rp.qs, kbx, S)), static_cast<const uint8_t*>(madw(rp.sc, kbx, S)),+ static_cast<const uint8_t*>(madw(rp.blk, kbx, S)), rp.iqs};+ }+ __device__ static WR loadr(const RP& rp, int dv) { // F::load's bytes and aux arithmetic, block dv past rp's+ constexpr int S = int(sizeof(F::Block));+ const auto* ql = reinterpret_cast<const int*>(rp.qs + dv * S);+ const auto* sc = rp.sc + dv * S;+ WR r;+ r.bq8_offset = 2 * ((rp.iqs / 2) / 4);+ r.v[0] = ldg_nc_s32(ql);+ r.v[1] = ldg_nc_s32(ql + 4);+ const uint32_t s0 = ldg_nc_u16(sc), s2 = ldg_nc_u16(sc + 4), s4 = ldg_nc_u16(sc + 8);+ const int dm = ldg_nc_s32(rp.blk + dv * S);+ const int j = r.bq8_offset / 2;+ const uint32_t hi = uint32_t(-int32_t(j >= 2));+ r.aux[0] = uint16_t(((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi));+ r.aux[1] = uint16_t(((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi));+ { half2 h; std::memcpy(&h, &dm, 4); r.dm = h; }+ return r;+ }+ __device__ static int xb(int iqs) { return 2 * ((iqs / 2) / 4); } // the lane's q8_1 block and int within a 256-chunk+ __device__ static int xp(int iqs) { return (iqs / 2) % 4; }+ template <int NC, bool FAST, class X>+ __device__ static void applyr(const WR& r, const X& x, const int* qb, const float* db, int kby, int iqs, float (&out)[NC]) {+ if constexpr (!FAST) {+ apply<NC>(r, x, kby, iqs, out);+ } else { // P2: q4_q8_dot_impl with (dot1 * sc), (dot2 * m) through i2f_small (|dot1 * sc| <= 15240 * 63, |dot2 * m| <= 1016 * 63)+ int u[4][NC];+ float d8[2][NC];+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ x.qat(qb + i * X::QB, u[2 * i]);+ x.qat(qb + i * X::QB + x.qp(4), u[2 * i + 1]);+ x.dat(db + i * X::DB, d8[i]);+ }+ const uint8_t* sc = reinterpret_cast<const uint8_t*>(r.aux);+ const uint8_t* m = sc + 2;+ const float2 dm4f = __half22float2(r.dm);+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ const int v0i = (r.v[0] >> (4 * i)) & 0x0f0f0f0f;+ const int v1i = (r.v[1] >> (4 * i)) & 0x0f0f0f0f;+ const int dot1 = STRATA_DP4A(v1i, u[2 * i + 1][c], STRATA_DP4A(v0i, u[2 * i][c], 0));+ const int dot2 = STRATA_DP4A(0x01010101, u[2 * i + 1][c], STRATA_DP4A(0x01010101, u[2 * i][c], 0));+ sumf_d += d8[i][c] * i2f_small(dot1 * sc[i] + kI2fMagic);+ sumf_m += d8[i][c] * i2f_small(dot2 * m[i] + kI2fMagic);+ }+ out[c] = dm4f.x * sumf_d - dm4f.y * sumf_m;+ }+ }+ }template <int NC, class X>__device__ static void apply(const F::W& r, const X& x, int kby, int iqs, float (&out)[NC]) {int u[4][NC];@@ -1583,6 +1779,78 @@ struct IlQ4K {struct IlQ5K {using F = Q5KTraits;static constexpr bool PM = false;+ static constexpr bool HAS_PF = true; // P1: as IlQ4K+ struct RP { const uint8_t *qs, *qh, *sc, *blk; int iqs; };+ __device__ static RP rowptrs(const F::Block* __restrict__ row, int iqs) {+ const auto* b = reinterpret_cast<const uint8_t*>(row);+ const int bq8_offset = 2 * ((iqs / 2) / 4), jm = (bq8_offset / 2) & 1;+ return {b + 48 + 16 * bq8_offset + 4 * ((iqs / 2) % 4), b + 16 + 4 * ((iqs / 2) % 4), b + 4 + 2 * jm, b, iqs};+ }+ using WR = F::W;+ __device__ static RP at(const RP& rp, int kbx) {+ constexpr int S = int(sizeof(F::Block));+ return {static_cast<const uint8_t*>(madw(rp.qs, kbx, S)), static_cast<const uint8_t*>(madw(rp.qh, kbx, S)),+ static_cast<const uint8_t*>(madw(rp.sc, kbx, S)), static_cast<const uint8_t*>(madw(rp.blk, kbx, S)), rp.iqs};+ }+ __device__ static WR loadr(const RP& rp, int dv) { // F::load's bytes and aux arithmetic, block dv past rp's+ constexpr int S = int(sizeof(F::Block));+ const auto* ql = reinterpret_cast<const int*>(rp.qs + dv * S);+ const auto* qh = reinterpret_cast<const int*>(rp.qh + dv * S);+ const auto* sc = rp.sc + dv * S;+ WR r;+ r.bq8_offset = 2 * ((rp.iqs / 2) / 4);+ r.vl[0] = ldg_nc_s32(ql);+ r.vl[1] = ldg_nc_s32(ql + 4);+ r.vh[0] = ldg_nc_s32(qh) >> r.bq8_offset;+ r.vh[1] = ldg_nc_s32(qh + 4) >> r.bq8_offset;+ const uint32_t s0 = ldg_nc_u16(sc), s2 = ldg_nc_u16(sc + 4), s4 = ldg_nc_u16(sc + 8);+ const int dm = ldg_nc_s32(rp.blk + dv * S);+ const int j = r.bq8_offset / 2;+ const uint32_t hi = uint32_t(-int32_t(j >= 2));+ r.aux[0] = uint16_t(((s0 & 0x3f3f) & ~hi) | ((((s4 >> 0) & 0x0f0f) | ((s0 & 0xc0c0) >> 2)) & hi));+ r.aux[1] = uint16_t(((s2 & 0x3f3f) & ~hi) | ((((s4 >> 4) & 0x0f0f) | ((s2 & 0xc0c0) >> 2)) & hi));+ { half2 h; std::memcpy(&h, &dm, 4); r.dm = h; }+ return r;+ }+ __device__ static int xb(int iqs) { return 2 * ((iqs / 2) / 4); }+ __device__ static int xp(int iqs) { return (iqs / 2) % 4; }+ template <int NC, bool FAST, class X>+ __device__ static void applyr(const WR& r, const X& x, const int* qb, const float* db, int kby, int iqs, float (&out)[NC]) {+ if constexpr (!FAST) {+ apply<NC>(r, x, kby, iqs, out);+ } else { // P2: q5_q8_dot_impl with (dot1 * sc), (dot2 * m) through i2f_small (|dot1 * sc| <= 31496 * 63 < 2^22)+ int u[4][NC];+ float d8[2][NC];+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ x.qat(qb + i * X::QB, u[2 * i]);+ x.qat(qb + i * X::QB + x.qp(4), u[2 * i + 1]);+ x.dat(db + i * X::DB, d8[i]);+ }+ const uint8_t* sc = reinterpret_cast<const uint8_t*>(r.aux);+ const uint8_t* m = sc + 2;+ const float2 dm5f = __half22float2(r.dm);+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ float sumf_d = 0.0f;+ float sumf_m = 0.0f;+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ const int vl0i = (r.vl[0] >> (4 * i)) & 0x0f0f0f0f;+ const int vl1i = (r.vl[1] >> (4 * i)) & 0x0f0f0f0f;+ const int vh0i = ((r.vh[0] >> i) << 4) & 0x10101010;+ const int vh1i = ((r.vh[1] >> i) << 4) & 0x10101010;+ const int v0i = vl0i | vh0i;+ const int v1i = vl1i | vh1i;+ const int dot1 = STRATA_DP4A(v0i, u[2 * i][c], STRATA_DP4A(v1i, u[2 * i + 1][c], 0));+ const int dot2 = STRATA_DP4A(0x01010101, u[2 * i][c], STRATA_DP4A(0x01010101, u[2 * i + 1][c], 0));+ sumf_d += d8[i][c] * i2f_small(dot1 * sc[i] + kI2fMagic);+ sumf_m += d8[i][c] * i2f_small(dot2 * m[i] + kI2fMagic);+ }+ out[c] = dm5f.x * sumf_d - dm5f.y * sumf_m;+ }+ }+ }template <int NC, class X>__device__ static void apply(const F::W& r, const X& x, int kby, int iqs, float (&out)[NC]) {int u[4][NC];@@ -1608,6 +1876,7 @@ struct IlQ5K {struct IlQ3K {using F = Q3KTraits;static constexpr bool PM = false;+ static constexpr bool HAS_PF = false; // the Q2_0 pack's type: keeps native_mmvq_il_kernel (F::W points into the block)template <int NC, class X>__device__ static void apply(const F::W& r, const X& x, int kby, int iqs, float (&out)[NC]) {int u[4][NC];@@ -1662,8 +1931,210 @@ struct IlQ6K {out[c] = q6_q8_dot_impl(r.vl, r.vh, uc, r.scales, r.d, dc);}}+ // P1: the two scale bytes loaded with the rest (F::W keeps a pointer into the block, which would put their loads+ // in the compute phase); q6_q8_dot_sc is q6_q8_dot_impl's expression+ static constexpr bool HAS_PF = true;+ // a row's four lane-adjusted byte pointers (ql, qh, scales, d): a block's loads are then one IMAD.WIDE each from kbx+ struct RP { const uint8_t *ql, *qh, *sc, *d; };+ __device__ static RP rowptrs(const F::Block* __restrict__ row, int iqs) {+ const auto* b = reinterpret_cast<const uint8_t*>(row);+ const int scale_offset = 8 * (iqs / 16) + (iqs % 16) / 4;+ return {b + 4 * iqs, b + 128 + 4 * (8 * (iqs / 16) + iqs % 8), b + 192 + scale_offset, b + 208}; // load_int_b2's int iqs+ }+ struct WR { int vl, vh, sc0, sc1, d; };+ __device__ static RP at(const RP& rp, int kbx) {+ constexpr int S = int(sizeof(F::Block));+ return {static_cast<const uint8_t*>(madw(rp.ql, kbx, S)), static_cast<const uint8_t*>(madw(rp.qh, kbx, S)),+ static_cast<const uint8_t*>(madw(rp.sc, kbx, S)), static_cast<const uint8_t*>(madw(rp.d, kbx, S))};+ }+ __device__ static WR loadr(const RP& rp, int dv) { // block dv past rp's: immediates off its four pointers+ constexpr int S = int(sizeof(F::Block));+ WR r;+ r.vl = ldg_nc_b2(rp.ql + dv * S, 0);+ r.vh = ldg_nc_b2(rp.qh + dv * S, 0);+ r.sc0 = ldg_nc_s8(rp.sc + dv * S);+ r.sc1 = ldg_nc_s8(rp.sc + dv * S + 4);+ r.d = ldg_nc_u16(rp.d + dv * S);+ return r;+ }+ __device__ static int xb(int iqs) { return 4 * (iqs / 16) + (iqs % 16) / 8; }+ __device__ static int xp(int iqs) { return iqs % 8; }+ template <int NC, bool FAST, class X>+ __device__ static void applyr(const WR& r, const X& x, const int* qb, const float* db, int kby, int iqs, float (&out)[NC]) {+ const int vh = r.vh >> (2 * ((iqs % 16) / 8));+ int u[2][NC];+ float d8[2][NC];+#pragma unroll+ for (int i = 0; i < 2; ++i) {+ x.qat(qb + 2 * i * X::QB, u[i]);+ x.dat(db + 2 * i * X::DB, d8[i]);+ }+ const float d = __half2float(__ushort_as_half((unsigned short) r.d));+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ const int uc[2] = {u[0][c], u[1][c]};+ const float dc[2] = {d8[0][c], d8[1][c]};+ out[c] = FAST ? q6_q8_dot_fast(r.vl, vh, uc, r.sc0, r.sc1, d, dc) : q6_q8_dot_sc(r.vl, vh, uc, r.sc0, r.sc1, d, dc);+ }+ }};+// P1: native_mmvq_il_kernel with every weight load of a group (the 4 virtual warps x R rows) issued+// before any of the group's dots - the four `if (kbx < bpr)` sections of the kernel below each sit behind their own+// branch, so a warp had one 2-row x 1-block group in flight (ncu: 432 GB/s on the qkv Q6_K matrix). The addresses are+// clamped to the row's last block (a tail re-reads it; its dot is skipped), and with DB the next group's loads are+// issued before this group's dots (two register sets, ping-pong, no copies). The per-lane accumulation order is the+// kernel below's: acc[v] += block(4m + v) in ascending m, ((acc0 + acc1) + acc2) + acc3, warp_sum - the same bits.+// STRATA_MMVQ_IL_PF=1: hoisted group loads (4 blocks a SM); =3: + P2's exact integer forms (the traits' applyr<FAST>);+// =4: as 3 at 5 blocks a SM; =2 / =5: as 3 / 4 with the ring (DB 2: a section's registers reloaded with the next group's+// blocks right after its dots); unset / 0: the kernel below.+template <typename I, int NC, int R, int DB, int MINB, bool FAST>+__launch_bounds__(4 * WARP, MINB)+__global__ void native_mmvq_il_pf_kernel(const typename I::F::Block* __restrict__ w, const int* __restrict__ xq,+ const float* __restrict__ xd, float* __restrict__ y, int n_in, int n_out) {+ using F = typename I::F;+ using WR = typename I::WR;+ constexpr int SUB = WARP / F::T; // weight blocks a chunk covers+ const int lane = int(threadIdx.x) & (WARP - 1);+ const auto x = q8_1_cols<NC, I::PM>(xq, xd, n_in / Q8K);+ using X = std::remove_const_t<decltype(x)>;+ const int bpr = n_in / F::DIV;+ const int kqs = F::kqs(lane);+ const int sub = lane / F::T;+ const int nm = (bpr + SUB - 1) / SUB;+ for (int grp = int(blockIdx.x) * 4 + (int(threadIdx.x) >> 5); grp * R < n_out; grp += int(gridDim.x) * 4) {+ const int row0 = grp * R;+ typename I::RP wr[R];+#pragma unroll+ for (int i = 0; i < R; ++i) wr[i] = I::rowptrs(w + std::size_t(min(row0 + i, n_out - 1)) * bpr, kqs); // a partial last group recomputes its last row+ float acc[4][R][NC];+#pragma unroll+ for (int v = 0; v < 4; ++v)+#pragma unroll+ for (int i = 0; i < R; ++i)+#pragma unroll+ for (int c = 0; c < NC; ++c) acc[v][i][c] = 0.0f;+ // the groups whose four chunks are whole blocks for every lane run with no branch at all (ptxas sinks a load+ // whose only use sits behind a branch into it, which is what put the scale loads back into the dot sections);+ // the tail group clamps its loads and predicates its dots+ const int m_full = bpr / SUB;+ // a full group's 4 x R blocks: one pointer set per row at the group's first block, the other three at immediates+ // (v * SUB blocks on); a clamped (tail) group addresses each block on its own+ auto load_group = [&](int m0, bool clamp, WR (&wv)[4][R]) {+ if (!clamp) {+#pragma unroll+ for (int i = 0; i < R; ++i) {+ const typename I::RP p0 = I::at(wr[i], SUB * m0 + sub);+#pragma unroll+ for (int v = 0; v < 4; ++v) wv[v][i] = I::loadr(p0, v * SUB);+ }+ } else {+#pragma unroll+ for (int v = 0; v < 4; ++v) {+ const int kbx = min(SUB * (m0 + v) + sub, bpr - 1);+#pragma unroll+ for (int i = 0; i < R; ++i) wv[v][i] = I::loadr(I::at(wr[i], kbx), 0);+ }+ }+ };+ // the lane's activation ints and scales of the group's first chunk; a section's are an immediate on+ const int* xq0 = x.qbase(I::xb(kqs) + sub * F::KBY, I::xp(kqs));+ const float* xd0 = x.dbase(I::xb(kqs) + sub * F::KBY);+ auto dot_group = [&](int m0, bool pred, const WR (&wv)[4][R]) {+ const int* qg = static_cast<const int*>(madw(xq0, SUB * m0 * F::KBY, X::QB * 4));+ const float* dg = static_cast<const float*>(madw(xd0, SUB * m0 * F::KBY, X::DB * 4));+#pragma unroll+ for (int v = 0; v < 4; ++v) {+ const int kbx = SUB * (m0 + v) + sub;+ if (!pred || kbx < bpr) {+#pragma unroll+ for (int i = 0; i < R; ++i) {+ float o[NC];+ I::template applyr<NC, FAST>(wv[v][i], x, qg + v * SUB * F::KBY * X::QB, dg + v * SUB * F::KBY * X::DB,+ kbx * F::KBY, kqs, o);+#pragma unroll+ for (int c = 0; c < NC; ++c) acc[v][i][c] += o[c];+ }+ }+ }+ };+ int m0 = 0;+ if constexpr (DB == 0) {+ for (; m0 + 4 <= m_full; m0 += 4) {+ WR wv[4][R];+ load_group(m0, false, wv);+ dot_group(m0, false, wv);+ }+ } else if constexpr (DB == 2) {+ // a ring: after a section's dots its registers take the next group's blocks of that section, so the warp+ // always has about a group's worth of loads in flight at the register cost of one group (the loads sit+ // behind a uniform branch whose consumers are in the next iteration, so nothing can sink)+ WR wv[4][R];+ if (m0 + 4 <= m_full) load_group(0, false, wv);+ for (; m0 + 4 <= m_full; m0 += 4) {+ const bool more = m0 + 8 <= m_full;+ typename I::RP pn[R];+ if (more) {+#pragma unroll+ for (int i = 0; i < R; ++i) pn[i] = I::at(wr[i], SUB * (m0 + 4) + sub);+ }+ const int* qg = static_cast<const int*>(madw(xq0, SUB * m0 * F::KBY, X::QB * 4));+ const float* dg = static_cast<const float*>(madw(xd0, SUB * m0 * F::KBY, X::DB * 4));+#pragma unroll+ for (int v = 0; v < 4; ++v) {+ const int kbx = SUB * (m0 + v) + sub;+#pragma unroll+ for (int i = 0; i < R; ++i) {+ float o[NC];+ I::template applyr<NC, FAST>(wv[v][i], x, qg + v * SUB * F::KBY * X::QB, dg + v * SUB * F::KBY * X::DB,+ kbx * F::KBY, kqs, o);+#pragma unroll+ for (int c = 0; c < NC; ++c) acc[v][i][c] += o[c];+ }+ if (more) {+#pragma unroll+ for (int i = 0; i < R; ++i) wv[v][i] = I::loadr(pn[i], v * SUB);+ }+ }+ }+ } else { // two register sets: a group's loads issue before the previous group's dots; no branch inside the loop+ WR wa[4][R], wb[4][R];+ if (m0 + 4 <= m_full) load_group(0, false, wa);+ for (; m0 + 12 <= m_full; m0 += 8) {+ load_group(m0 + 4, false, wb);+ dot_group(m0, false, wa);+ load_group(m0 + 8, false, wa);+ dot_group(m0 + 4, false, wb);+ }+ if (m0 + 8 <= m_full) {+ load_group(m0 + 4, false, wb);+ dot_group(m0, false, wa);+ dot_group(m0 + 4, false, wb);+ m0 += 8;+ } else if (m0 + 4 <= m_full) {+ dot_group(m0, false, wa);+ m0 += 4;+ }+ }+ if (m0 < nm) {+ WR wv[4][R];+ load_group(m0, true, wv);+ dot_group(m0, true, wv);+ }+#pragma unroll+ for (int i = 0; i < R; ++i)+#pragma unroll+ for (int c = 0; c < NC; ++c) {+ float s = acc[0][i][c];+ s += acc[1][i][c];+ s += acc[2][i][c];+ s += acc[3][i][c];+ s = warp_sum(s);+ if (lane == 0 && row0 + i < n_out) y[std::size_t(c) * n_out + row0 + i] = s;+ }+ }+}+template <typename I, int NC, int R>__launch_bounds__(4 * WARP)__global__ void native_mmvq_il_kernel(const typename I::F::Block* __restrict__ w, const int* __restrict__ xq,@@ -1717,23 +2188,57 @@ __global__ void native_mmvq_il_kernel(const typename I::F::Block* __restrict__ w}}+int g_il_pf = -1; // native_mmvq_il_pf (tests, benchmarks): the variant for every launch; -1: STRATA_MMVQ_IL_PF+int il_pf_env() {+ static const int env = [] { const char* v = std::getenv("STRATA_MMVQ_IL_PF"); return v ? std::atoi(v) : 0; }();+ return env;+}+int il_pf_table(int type, int ncols, int n_out); // the sm_80 table below: pf << 4 | rows, 0 = today's kernel and table+// the variant of one launch (g_il_pf, else STRATA_MMVQ_IL_PF: 0 / unset today's kernel, 1 the measured table, 2-6 that+// variant for every type and shape), and with the table its rows a warp (0: the rows table's)+int il_pf_for(int type, int ncols, int n_out, int* rows) {+ *rows = 0;+ if (g_il_pf >= 0) return g_il_pf;+ const int env = il_pf_env();+ if (env != 1) return env;+ const int t = il_pf_table(type, ncols, n_out);+ *rows = t & 15;+ return t >> 4;+}+// one (type, columns, rows) launch: today's kernel, or the variant il_pf_for names for a type that has them+template <typename I, int NC, int R>+void launch_il_one(unsigned blocks, const typename I::F::Block* w, const int* xq, const float* xd, float* y, int n_in, int n_out,+ cudaStream_t s, int pf) {+ if constexpr (I::HAS_PF) {+ switch (pf) {+ case 6: native_mmvq_il_pf_kernel<I, NC, R, 0, 4, false><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); return;+ case 2: native_mmvq_il_pf_kernel<I, NC, R, 2, 4, true><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); return;+ case 3: native_mmvq_il_pf_kernel<I, NC, R, 0, 4, true><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); return;+ case 4: native_mmvq_il_pf_kernel<I, NC, R, 0, 5, true><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); return;+ case 5: native_mmvq_il_pf_kernel<I, NC, R, 2, 5, true><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); return;+ default: break;+ }+ }+ native_mmvq_il_kernel<I, NC, R><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out);+}+template <typename I, int R>-void launch_il(const void* weights, const void* x_il, float* y, int n_in, int n_out, int ncols, cudaStream_t s) {+void launch_il(const void* weights, const void* x_il, float* y, int n_in, int n_out, int ncols, cudaStream_t s, int pf) {const auto* w = static_cast<const typename I::F::Block*>(weights);const unsigned blocks = unsigned(((n_out + R - 1) / R + 3) / 4);if (ncols == 1) { // 143-dense: one column reads the plain q8_1 blocks (x_il is then the caller's x_q8_1)- native_mmvq_il_kernel<I, 1, R><<<blocks, 4 * WARP, 0, s>>>(w, static_cast<const int*>(x_il), nullptr, y, n_in, n_out);+ launch_il_one<I, 1, R>(blocks, w, static_cast<const int*>(x_il), nullptr, y, n_in, n_out, s, pf);return;}const Q81IlParts parts = q8_1_il_parts(x_il, n_in, ncols);const int* xq = I::PM ? parts.pm : parts.bm;const float* xd = parts.d;switch (ncols) {-#define STRATA_IL_CASE(N) case N: native_mmvq_il_kernel<I, N, R><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); break;+#define STRATA_IL_CASE(N) case N: launch_il_one<I, N, R>(blocks, w, xq, xd, y, n_in, n_out, s, pf); break;STRATA_IL_CASE(2) STRATA_IL_CASE(3) STRATA_IL_CASE(4)#undef STRATA_IL_CASE// 147-verify: 5..8 columns (the cp = 8 copy) at 1 or 2 rows a warp; launch_il_rows maps 4 rows to 2 there-#define STRATA_IL_CASE(N) case N: if constexpr (R <= 2) native_mmvq_il_kernel<I, N, R><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); break;+#define STRATA_IL_CASE(N) case N: if constexpr (R <= 2) launch_il_one<I, N, R>(blocks, w, xq, xd, y, n_in, n_out, s, pf); break;STRATA_IL_CASE(5) STRATA_IL_CASE(6) STRATA_IL_CASE(7) STRATA_IL_CASE(8)#undef STRATA_IL_CASEdefault: throw std::invalid_argument("native_mmvq_il requires 1 <= ncols <= 8");@@ -1742,11 +2247,14 @@ void launch_il(const void* weights, const void* x_il, float* y, int n_in, int n_template <typename I>void launch_il_rows(int r, const void* weights, const void* x_il, float* y, int n_in, int n_out, int ncols, cudaStream_t s) {+ int pf_rows = 0;+ const int pf = il_pf_for(I::F::TYPE, ncols, n_out, &pf_rows);+ if (pf_rows != 0) r = pf_rows; // the pf table measured its rows with its kernelif (ncols > 4 && r == 4) r = 2; // 5..8 columns: 1 or 2 rows a warpswitch (r) {- case 1: launch_il<I, 1>(weights, x_il, y, n_in, n_out, ncols, s); break;- case 4: launch_il<I, 4>(weights, x_il, y, n_in, n_out, ncols, s); break;- default: launch_il<I, 2>(weights, x_il, y, n_in, n_out, ncols, s); break;+ case 1: launch_il<I, 1>(weights, x_il, y, n_in, n_out, ncols, s, pf); break;+ case 4: launch_il<I, 4>(weights, x_il, y, n_in, n_out, ncols, s, pf); break;+ default: launch_il<I, 2>(weights, x_il, y, n_in, n_out, ncols, s, pf); break;}}@@ -3054,6 +3562,20 @@ constexpr IlRows kIlRows_80[] = {{14, {{0, 0, 2, 2, 2}, {0, 2, 2, 2, 2}, {0, 2, 4, 2, 2}}}, // Q6_K{11, {{1, 2, 2, 2, 2}, {1, 1, 2, 2, 2}, {1, 2, 2, 2, 2}}}, // Q3_K};+// sm_80 P1/P2: the hoisted-load / exact-integer kernel variant and its rows a warp per cell, pf << 4 | rows+// (0x32 = variant 3 at 2 rows a warp; 0 = today's kernel with kIlRows_80), read off mmvq_il_parity --bench --emit-table+// --pf 0,3,4 on the CMP 170HX with the same 3% rule against native_mmvq; STRATA_MMVQ_IL_PF=1 takes it. Every variant is+// bitwise native_mmvq (the tool checks every case, T 1-8, rows 1/2/4, each variant).+struct IlPf { int type; uint8_t v[3][5]; };+struct IlPf58 { int type; uint8_t v[4][5]; };+// Only Q6_K: the same variants are bitwise and measured for Q4_K / Q5_K / IQ4_XS too, but there they tie today's kernel+// within the run-to-run noise (their emitted cells flip between 0 and a variant), so those types keep kIlRows_80.+constexpr IlPf kIlPf_80[] = {+ {14, {{0x31, 0x42, 0x32, 0x42, 0x42}, {0x31, 0x41, 0x32, 0x32, 0x41}, {0x31, 0x41, 0x32, 0x32, 0x32}}}, // Q6_K+};+constexpr IlPf58 kIlPf58_80[] = { // (rows 4 runs as 2 at 5+ columns, so the emitted 0x34 cells are written 0x32)+ {14, {{0x31, 0x42, 0x32, 0x32, 0x32}, {0x31, 0x31, 0x32, 0x32, 0x32}, {0x31, 0x31, 0x02, 0x32, 0x02}, {0x31, 0x31, 0x31, 0x31, 0x31}}}, // Q6_K+};// Turing (sm_75): read off mmvq_il_parity --bench --emit-table on an RTX 2080 Ti 22 GB (SM 1650 MHz), the same 3% rule.// The plain multi-column kernels are bound by dp4a throughput on Turing from 2 columns on (IQ4_XS 2560x10240 at 4// columns 81.1 -> 37.4 us, the Q5_K head 1671 -> 1045 us); bitwise equal to native_mmvq. No one-column table here:@@ -3220,9 +3742,22 @@ int il_cc() {return cc[dev];}int il_rows(int type, int ncols, int n_out) { return native_mmvq_il_rows_for(il_cc(), type, ncols, n_out); }+int il_pf_table(int type, int ncols, int n_out) {+ if (il_cc() != 80 || ncols < 2 || ncols > 8) return 0; // only the CMP 170HX measured one+ const int cls = n_out < 2048 ? 0 : n_out < 4096 ? 1 : n_out < 8192 ? 2 : n_out < 12288 ? 3 : 4;+ if (ncols <= 4) {+ for (const IlPf& t : kIlPf_80)+ if (t.type == type) return t.v[ncols - 2][cls];+ } else {+ for (const IlPf58& t : kIlPf58_80)+ if (t.type == type) return t.v[ncols - 5][cls];+ }+ return 0;+}int g_tune_rows = 0; // native_mmvq_il_tune (tests, benchmarks): rows a warp for every shape (0: the table)} // namespacevoid native_mmvq_il_tune(int rows) { g_tune_rows = rows; }+void native_mmvq_il_pf(int pf) { g_il_pf = pf; }namespace {// sm_80 and newer (measured on sm_86 and sm_120), Volta (sm_70) and Pascal (sm_60, sm_61), each with its own table@@ -3298,6 +3833,7 @@ void native_mmvq_il(int ggml_type, const void* weights, const void* x_q8_1, consnative_mmvq(ggml_type, weights, x_q8_1, y, n_in, n_out, ncols, stream);}void native_mmvq_il_tune(int) {}+void native_mmvq_il_pf(int) {}#endif // !__HIPCC__} // namespace strata::kernelsdiff --git a/src/kernels/cuda/q8_1_il.cuh b/src/kernels/cuda/q8_1_il.cuhindex 49c7a5a1f..4d64fe07b 100644--- a/src/kernels/cuda/q8_1_il.cuh+++ b/src/kernels/cuda/q8_1_il.cuh@@ -41,6 +41,52 @@ template <int NC, bool PM> struct Q81Il {}}}+ // P1 (native_mmvq_il_pf_kernel): the same loads from a pointer the caller strength-reduces - qat(qbase(b, p) + k * QB)+ // is u(b + k, p), dat(dbase(b) + k * DB) is scales(b + k)+ static constexpr int QB = PM ? CP : 8 * CP, DB = CP;+ __device__ __forceinline__ int qp(int dp) const { return PM ? dp * nb * CP : dp * CP; } // ints from p to p + dp+ __device__ __forceinline__ const int* qbase(int b, int p) const { return q + std::size_t(PM ? p * nb + b : b * 8 + p) * CP; }+ __device__ __forceinline__ const float* dbase(int b) const { return d + std::size_t(b) * CP; }+ __device__ __forceinline__ void qat(const int* s, int (&o)[NC]) const { // __ldg: s comes from an asm (madw), so the+ if constexpr (CP == 2) { // front end no longer knows it is global+ const int2 a = __ldg(reinterpret_cast<const int2*>(s));+ o[0] = a.x;+ o[1] = a.y;+ } else {+ const int4 a = __ldg(reinterpret_cast<const int4*>(s));+ o[0] = a.x;+ if constexpr (NC > 1) o[1] = a.y;+ if constexpr (NC > 2) o[2] = a.z;+ if constexpr (NC > 3) o[3] = a.w;+ if constexpr (NC > 4) {+ const int4 e = __ldg(reinterpret_cast<const int4*>(s + 4));+ o[4] = e.x;+ if constexpr (NC > 5) o[5] = e.y;+ if constexpr (NC > 6) o[6] = e.z;+ if constexpr (NC > 7) o[7] = e.w;+ }+ }+ }+ __device__ __forceinline__ void dat(const float* s, float (&o)[NC]) const {+ if constexpr (CP == 2) {+ const float2 a = __ldg(reinterpret_cast<const float2*>(s));+ o[0] = a.x;+ o[1] = a.y;+ } else {+ const float4 a = __ldg(reinterpret_cast<const float4*>(s));+ o[0] = a.x;+ if constexpr (NC > 1) o[1] = a.y;+ if constexpr (NC > 2) o[2] = a.z;+ if constexpr (NC > 3) o[3] = a.w;+ if constexpr (NC > 4) {+ const float4 e = __ldg(reinterpret_cast<const float4*>(s + 4));+ o[4] = e.x;+ if constexpr (NC > 5) o[5] = e.y;+ if constexpr (NC > 6) o[6] = e.z;+ if constexpr (NC > 7) o[7] = e.w;+ }+ }+ }__device__ __forceinline__ void scales(int b, float (&o)[NC]) const {const float* s = d + std::size_t(b) * CP;if constexpr (CP == 2) {@@ -73,6 +119,12 @@ struct Q81One {__device__ __forceinline__ void scales(int b, float (&o)[1]) const {o[0] = __low2float(*reinterpret_cast<const half2*>(x + std::size_t(b) * 36));}+ static constexpr int QB = 9, DB = 9; // a 36-byte block in ints / floats+ __device__ __forceinline__ int qp(int dp) const { return dp; }+ __device__ __forceinline__ const int* qbase(int b, int p) const { return reinterpret_cast<const int*>(x + std::size_t(b) * 36 + 4) + p; }+ __device__ __forceinline__ const float* dbase(int b) const { return reinterpret_cast<const float*>(x + std::size_t(b) * 36); }+ __device__ __forceinline__ void qat(const int* s, int (&o)[1]) const { o[0] = __ldg(s); }+ __device__ __forceinline__ void dat(const float* s, float (&o)[1]) const { o[0] = __low2float(__ldg(reinterpret_cast<const half2*>(s))); }};// The activations a kernel for NC columns reads: one column's q8_1 blocks, 2+ columns' interleaved copy (nb blocks adiff --git a/src/kernels/mmvq_il_parity.cpp b/src/kernels/mmvq_il_parity.cppindex 415baa4be..e55d6d013 100644--- a/src/kernels/mmvq_il_parity.cpp+++ b/src/kernels/mmvq_il_parity.cpp@@ -1,7 +1,13 @@// src/kernels/mmvq_il_parity.cpp - native_mmvq_il (2-4 columns from the interleaved q8_1 copy, fork F4) against// native_mmvq's multi-column kernels, bitwise, for every rows-a-warp choice; --bench times both per call.//-// build/mmvq_il_parity [--bench [--emit-table] [--hot] [--gap MB] [--persist] [--ab]]+// build/mmvq_il_parity [--bench [--emit-table] [--hot] [--gap MB] [--persist] [--ab] [--pf a,b,..] [--rounds N]]+//+// --pf a,b,..: the P1 / P2 kernel variants (native_mmvq_il_pf: 0 = today's kernel, 6 hoisted group loads, 3 + P2's exact+// integer forms, 4 = 3 at 5 blocks a SM, 2 / 5 = 3 / 4 with a ring) are each bit-checked against native_mmvq, and with+// --bench timed interleaved per (case, T, rows): N rounds+// (default 5) of 100 calls each, variant after variant, the median of the rounds printed per variant (PFBENCH lines; the+// table cells take the first listed variant). Without --pf every call takes STRATA_MMVQ_IL_PF (unset: today's kernel).//// --hot times one copy of the weights (they sit in L2 after the first call: the ceiling of an L2// prefetch); --gap MB reads MB megabytes of OTHER weights (native_mmvq over other copies) before every timed call, as@@ -23,6 +29,7 @@#include <cuda_runtime.h>#include <algorithm>+#include <array>#include <cmath>#include <cstdint>#include <cstdio>@@ -35,9 +42,11 @@namespace {using namespace strata::kernels;-struct Cell { int type, T, n_out; double ref; double il[5]; };+struct Cell { int type, T, n_out, pf; double ref; double il[5]; }; // pf: the variant the il[] times ran (-1: the environment's)std::vector<Cell> g_cells;bool g_hot = false, g_persist = false, g_ab = false;+std::vector<int> g_pf; // --pf: the variants to check and time; empty: the environment's+int g_rounds = 5;std::size_t g_gap_bytes = 0;std::size_t g_gap_now = 0; // the gap in force for the current arm (--ab) or g_gap_bytes// time `reps` calls of `fn` on `s`; with --gap each call is preceded by `gap` (other weights' reads) and timed on its own@@ -195,6 +204,7 @@ int run_case(const Case& c, bool bench, cudaStream_t s) {std::vector<uint32_t> a((std::size_t) T * c.n_out), b(a.size());ck(cudaMemcpy(a.data(), yref, ybytes, cudaMemcpyDeviceToHost), "read ref");double best_ref = 0, best_il[5] = {0, 0, 0, 0, 0};+ std::vector<std::array<double, 5>> pf_il(g_pf.size(), std::array<double, 5>{0, 0, 0, 0, 0}); // per --pf variantint table_r = 0;struct Arm { const char* name; bool hot; std::size_t gap; bool persist; };const Arm arms_ab[] = {{"cold", false, 0, false}, {"hot", true, 0, false}, {"hot+gap", true, g_gap_bytes, false},@@ -208,34 +218,60 @@ int run_case(const Case& c, bool bench, cudaStream_t s) {g_gap_now = arms[ai].gap;if (arms[ai].persist != g_persist) { g_persist = arms[ai].persist; persist_window(s, dw[0], wbytes, g_persist); }}+ const std::vector<int> pfs = g_pf.empty() ? std::vector<int>{-1} : g_pf;for (int r : {0, 1, 2, 4}) {native_mmvq_il_tune(r);- cudaMemset(yil, 0xff, ybytes);- native_mmvq_il(c.type, dw[0], xq, xil, yil, c.n_in, c.n_out, T, s);- ck(cudaStreamSynchronize(s), "il");- ck(cudaMemcpy(b.data(), yil, ybytes, cudaMemcpyDeviceToHost), "read il");- long long diff = 0, nonfinite = 0;- for (std::size_t i = 0; i < a.size(); ++i) {- float fa;- std::memcpy(&fa, &a[i], 4);- if (!std::isfinite(fa)) ++nonfinite;- if (a[i] != b[i]) ++diff;- }- const bool used = r != 0 || native_mmvq_il_supported(c.type, T, c.n_out);- if (diff != 0 || nonfinite != 0) {- ++bad;- std::printf("FAIL %-26s T=%d rows=%d: %lld bits differ, %lld non-finite of %zu\n", c.name, T, r, diff, nonfinite, a.size());+ for (int pf : pfs) {+ native_mmvq_il_pf(pf);+ cudaMemset(yil, 0xff, ybytes);+ native_mmvq_il(c.type, dw[0], xq, xil, yil, c.n_in, c.n_out, T, s);+ ck(cudaStreamSynchronize(s), "il");+ ck(cudaMemcpy(b.data(), yil, ybytes, cudaMemcpyDeviceToHost), "read il");+ long long diff = 0, nonfinite = 0;+ for (std::size_t i = 0; i < a.size(); ++i) {+ float fa;+ std::memcpy(&fa, &a[i], 4);+ if (!std::isfinite(fa)) ++nonfinite;+ if (a[i] != b[i]) ++diff;+ }+ if (diff != 0 || nonfinite != 0) {+ ++bad;+ std::printf("FAIL %-26s T=%d rows=%d pf=%d: %lld bits differ, %lld non-finite of %zu\n", c.name, T, r, pf, diff,+ nonfinite, a.size());+ }}- (void) used;+ native_mmvq_il_pf(-1);if (bench && r != 0) {- best_il[r] = time_us([&](int i) { native_mmvq_il(c.type, dw[g_hot ? 0 : i % copies], xq, xil, yil, c.n_in, c.n_out, T, s); }, 200, s, gap);+ auto il_call = [&](int i) { native_mmvq_il(c.type, dw[g_hot ? 0 : i % copies], xq, xil, yil, c.n_in, c.n_out, T, s); };+ if (g_pf.empty()) {+ best_il[r] = time_us(il_call, 200, s, gap);+ } else { // the variants interleaved, round after round; the median of each one's rounds+ std::vector<std::vector<double>> t(g_pf.size());+ for (int round = 0; round < g_rounds; ++round)+ for (std::size_t k = 0; k < g_pf.size(); ++k) {+ native_mmvq_il_pf(g_pf[k]);+ t[k].push_back(time_us(il_call, 100, s, gap));+ }+ native_mmvq_il_pf(-1);+ std::printf("PFBENCH %-26s T=%d rows=%d:", c.name, T, r);+ for (std::size_t k = 0; k < g_pf.size(); ++k) {+ std::sort(t[k].begin(), t[k].end());+ const double med = t[k][t[k].size() / 2];+ if (k == 0) best_il[r] = med;+ pf_il[k][r] = med;+ std::printf(" pf%d %.1f us (min %.1f max %.1f)", g_pf[k], med, t[k].front(), t[k].back());+ }+ std::printf("\n");+ }}}native_mmvq_il_tune(0);if (bench) {best_ref = time_us([&](int i) { native_mmvq(c.type, dw[g_hot ? 0 : i % copies], xq, yref, c.n_in, c.n_out, T, s); }, 200, s, gap);(void) table_r;- g_cells.push_back({c.type, T, c.n_out, best_ref, {0, best_il[1], best_il[2], 0, best_il[4]}});+ if (g_pf.empty()) g_cells.push_back({c.type, T, c.n_out, -1, best_ref, {0, best_il[1], best_il[2], 0, best_il[4]}});+ for (std::size_t k = 0; k < g_pf.size(); ++k)+ g_cells.push_back({c.type, T, c.n_out, g_pf[k], best_ref, {0, pf_il[k][1], pf_il[k][2], 0, pf_il[k][4]}});std::printf("BENCH %-26s T=%d %-16s multi %.1f us | il rows1 %.1f rows2 %.1f rows4 %.1f us\n", c.name, T,arms[ai].name, best_ref, best_il[1], best_il[2], best_il[4]);}@@ -256,33 +292,37 @@ void emit_table() {std::printf("// measured on this card by mmvq_il_parity --bench --emit-table\n");// per cell: the rows count whose WORST ratio to native_mmvq over the cell's measured shapes is the best, taken only// when that worst ratio is under 0.97 (else 0); a class with no measured shape takes its lower neighbour's value- auto pick_cell = [&](int type, int nc, int cls, int prev) {+ // pf_sel: the cells of that variant (the environment's when --pf was not given, else 0 = today's kernel); -2: every+ // listed variant, the answer then pf << 4 | rows (the kIlPf form), 0 when nothing beats native_mmvq by 3%+ auto pick_cell = [&](int type, int nc, int cls, int prev, int pf_sel) {double best_worst = 1e9;int best = -1;bool any = false;- for (int r : {1, 2, 4}) {- double worst = 0;- bool have = false;- for (const Cell& c : g_cells) {- if (c.type != type || c.T != nc) continue;- const int cc = c.n_out < 2048 ? 0 : c.n_out < 4096 ? 1 : c.n_out < 8192 ? 2 : c.n_out < 12288 ? 3 : 4;- if (cc != cls || c.il[r] <= 0) continue;- have = true;- worst = std::max(worst, c.il[r] / c.ref);+ for (int pf : (pf_sel == -2 ? (g_pf.empty() ? std::vector<int>{-1} : g_pf) : std::vector<int>{pf_sel}))+ for (int r : {1, 2, 4}) {+ double worst = 0;+ bool have = false;+ for (const Cell& c : g_cells) {+ if (c.type != type || c.T != nc || c.pf != pf) continue;+ const int cc = c.n_out < 2048 ? 0 : c.n_out < 4096 ? 1 : c.n_out < 8192 ? 2 : c.n_out < 12288 ? 3 : 4;+ if (cc != cls || c.il[r] <= 0) continue;+ have = true;+ worst = std::max(worst, c.il[r] / c.ref);+ }+ any = any || have;+ if (have && worst < best_worst) { best_worst = worst; best = pf_sel == -2 ? (std::max(pf, 0) << 4 | r) : r; }}- any = any || have;- if (have && worst < best_worst) { best_worst = worst; best = r; }- }if (!any) return prev;return best_worst < 0.97 ? best : 0;};+ const int pf_today = g_pf.empty() ? -1 : 0;for (int ti = 0; ti < 5; ++ti) {std::printf(" {%d, {", types[ti]);for (int nc = 2; nc <= 4; ++nc) {int prev = 0;std::printf("{");for (int cls = 0; cls < 5; ++cls) {- prev = pick_cell(types[ti], nc, cls, prev);+ prev = pick_cell(types[ti], nc, cls, prev, pf_today);std::printf("%d%s", prev, cls < 4 ? ", " : "");}std::printf("}%s", nc < 4 ? ", " : "");@@ -296,19 +336,49 @@ void emit_table() {int prev = 0;std::printf("{");for (int cls = 0; cls < 5; ++cls) {- prev = pick_cell(types[ti], nc, cls, prev);+ prev = pick_cell(types[ti], nc, cls, prev, pf_today);std::printf("%d%s", prev, cls < 4 ? ", " : "");}std::printf("}%s", nc < 8 ? ", " : "");}std::printf("}}, // %s\n", names[ti]);}+ if (!g_pf.empty()) { // the joint (variant, rows) table over the listed variants: kIlPf_80 / kIlPf58_80+ std::printf("// variant << 4 | rows over --pf (kIlPf_<cc>), 2-4 columns:\n");+ for (int ti = 0; ti < 5; ++ti) {+ std::printf(" {%d, {", types[ti]);+ for (int nc = 2; nc <= 4; ++nc) {+ int prev = 0;+ std::printf("{");+ for (int cls = 0; cls < 5; ++cls) {+ prev = pick_cell(types[ti], nc, cls, prev, -2);+ std::printf("0x%02x%s", prev, cls < 4 ? ", " : "");+ }+ std::printf("}%s", nc < 4 ? ", " : "");+ }+ std::printf("}}, // %s\n", names[ti]);+ }+ std::printf("// 5-8 columns (kIlPf58_<cc>):\n");+ for (int ti = 0; ti < 5; ++ti) {+ std::printf(" {%d, {", types[ti]);+ for (int nc = 5; nc <= 8; ++nc) {+ int prev = 0;+ std::printf("{");+ for (int cls = 0; cls < 5; ++cls) {+ prev = pick_cell(types[ti], nc, cls, prev, -2);+ std::printf("0x%02x%s", prev, cls < 4 ? ", " : "");+ }+ std::printf("}%s", nc < 8 ? ", " : "");+ }+ std::printf("}}, // %s\n", names[ti]);+ }+ }std::printf("// one column (kIl1RowsShared):\n");for (int ti = 0; ti < 5; ++ti) {std::printf(" {%d, {", types[ti]);int prev = 0;for (int cls = 0; cls < 5; ++cls) {- prev = pick_cell(types[ti], 1, cls, prev);+ prev = pick_cell(types[ti], 1, cls, prev, pf_today);std::printf("%d%s", prev, cls < 4 ? ", " : "");}std::printf("}}, // %s\n", names[ti]);@@ -342,6 +412,26 @@ int check_tables() {++bad;std::printf("FAIL table: the sm_60 table is not the P100's\n");}+ // sm_80 has its own tables (kIlRows_80 / kIlRows58_80 / kIl1Rows_80: a CMP 170HX's); the key 81 is what STRATA_MMVQ_IL_SM80=0+ // gives an sm_80 card: no table of its own, so the shared one (and no one-column table). Every answer is 0, 1, 2 or 4.+ {+ bool differs = false;+ for (int type : {23, 12, 13, 14, 11})+ for (int nc = 1; nc <= 8; ++nc)+ for (int n_out : {512, 2560, 6144, 10240, 12288, 248320}) {+ const int a = native_mmvq_il_rows_for(80, type, nc, n_out), b = native_mmvq_il_rows_for(81, type, nc, n_out);+ if (a != 0 && a != 1 && a != 2 && a != 4) { ++bad; std::printf("FAIL table: sm_80 rows %d\n", a); }+ differs = differs || a != b;+ }+ if (!differs) { ++bad; std::printf("FAIL table: sm_80 answers like the shared table\n"); }+ // spot checks read off the source tables: Q6_K at 3 columns and 2560 rows (class 1) is {0, 2, 2, 2, 2}[1] = 2; one-column+ // IQ4_XS at 6144 rows (class 2) is {0, 0, 1, 4, 1}[2] = 1 on sm_80 and 0 (no one-column table) on the key 81+ if (native_mmvq_il_rows_for(80, 14, 3, 2560) != 2 || native_mmvq_il_rows_for(80, 23, 1, 6144) != 1 ||+ native_mmvq_il_rows_for(81, 23, 1, 6144) != 0) {+ ++bad;+ std::printf("FAIL table: the sm_80 table is not the CMP 170HX's\n");+ }+ }if (native_mmvq_il_rows_for(89, 99, 2, 2560) != 0 || native_mmvq_il_rows_for(89, 12, 5, 2560) != 0 ||native_mmvq_il_rows_for(86, 12, 5, 2560) != 0 || native_mmvq_il_rows_for(120, 12, 9, 2560) != 0) {++bad;@@ -354,12 +444,22 @@ int check_tables() {int main(int argc, char** argv) {const bool bench = argc > 1 && std::string(argv[1]) == "--bench";const bool emit = bench && argc > 2 && std::string(argv[2]) == "--emit-table";- for (int i = 2; i < argc; ++i) {+ for (int i = 1; i < argc; ++i) {const std::string a = argv[i];if (a == "--hot") g_hot = true;else if (a == "--ab") g_ab = true;else if (a == "--persist") g_persist = true;else if (a == "--gap" && i + 1 < argc) g_gap_bytes = (std::size_t) std::atoi(argv[++i]) << 20;+ else if (a == "--rounds" && i + 1 < argc) g_rounds = std::atoi(argv[++i]);+ else if (a == "--pf" && i + 1 < argc) {+ const std::string list = argv[++i];+ for (std::size_t p = 0; p < list.size();) {+ std::size_t q = list.find(',', p);+ if (q == std::string::npos) q = list.size();+ g_pf.push_back(std::atoi(list.substr(p, q - p).c_str()));+ p = q + 1;+ }+ }}cudaStream_t s;cudaStreamCreate(&s);