Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
mmvq_il: 5-8 column interleaved kernels and the sm_120 tables (bitwise)
Niko122175fabbd94152
diff --git a/src/core/verify.cpp b/src/core/verify.cppindex 427c712ac..52a96be44 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -765,7 +765,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&lo_ = b.take<float>(T * (uint64_t) g.hc_lr); rs_ = b.take<float>(T * HC); xn_ = b.take<float>(T * HC * N);xq_ = b.take<uint8_t>(strata::kernels::native_q8_1_bytes(max_in, (int) T));sh_xq_ = b.take<uint8_t>(strata::kernels::native_q8_1_bytes(max_in, (int) T));- xil_ = b.take<uint8_t>(strata::kernels::native_q8_1_il_bytes(max_in, 4));+ xil_ = b.take<uint8_t>(strata::kernels::native_q8_1_il_bytes(max_in, std::max(4, (int) T))); // 147-verify: 5-8 columns tooqkv_L_ = b.take<float>(nG * T * C); h_L_ = b.take<float>(nG * T * C);gate_L_ = b.take<float>(nG * T * HV); beta_L_ = b.take<float>(nG * T * HV);z_ = b.take<float>(T * ZV); y_ = b.take<float>(T * ZV); y_dummy_ = b.take<float>(T * ZV);diff --git a/src/kernels/cuda/native_mmvq.cu b/src/kernels/cuda/native_mmvq.cuindex 2e3af20cd..ec0da97bb 100644--- a/src/kernels/cuda/native_mmvq.cu+++ b/src/kernels/cuda/native_mmvq.cu@@ -1730,12 +1730,17 @@ void launch_il(const void* weights, const void* x_il, float* y, int n_in, int n_#define STRATA_IL_CASE(N) case N: native_mmvq_il_kernel<I, N, R><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); break;STRATA_IL_CASE(2) STRATA_IL_CASE(3) STRATA_IL_CASE(4)#undef STRATA_IL_CASE- default: throw std::invalid_argument("native_mmvq_il requires 1 <= ncols <= 4");+ // 147-verify: 5..8 columns (the cp = 8 copy) at 1 or 2 rows a warp; launch_il_rows maps 4 rows to 2 there+#define STRATA_IL_CASE(N) case N: if constexpr (R <= 2) native_mmvq_il_kernel<I, N, R><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); break;+ STRATA_IL_CASE(5) STRATA_IL_CASE(6) STRATA_IL_CASE(7) STRATA_IL_CASE(8)+#undef STRATA_IL_CASE+ default: throw std::invalid_argument("native_mmvq_il requires 1 <= ncols <= 8");}}template <typename I>void launch_il_rows(int r, const void* weights, const void* x_il, float* y, int n_in, int n_out, int ncols, cudaStream_t s) {+ if (ncols > 4 && r == 4) r = 2; // 5..8 columns: 1 or 2 rows a warpswitch (r) {case 1: launch_il<I, 1>(weights, x_il, y, n_in, n_out, ncols, s); break;case 4: launch_il<I, 4>(weights, x_il, y, n_in, n_out, ncols, s); break;@@ -2956,7 +2961,7 @@ void native_quantize_q8_1_il(const float* x, void* x_q8_1, void* x_il, int n_in,void native_q8_1_interleave(const void* x_q8_1, void* x_il, int n_in, int ncols, void* stream) {validate_shape(n_in, ncols);- if (ncols < 2 || ncols > 4) throw std::invalid_argument("native_q8_1_interleave requires 2 <= ncols <= 4");+ if (ncols < 2 || ncols > 8) throw std::invalid_argument("native_q8_1_interleave requires 2 <= ncols <= 8");validate_pointer(x_q8_1);validate_pointer(x_il);validate_stream(stream);@@ -3028,6 +3033,16 @@ constexpr IlRows kIlRows_86[] = {{14, {{0, 0, 1, 1, 0}, {0, 0, 1, 1, 2}, {0, 0, 1, 1, 1}}}, // Q6_K{11, {{1, 2, 2, 2, 2}, {1, 1, 1, 2, 2}, {1, 1, 1, 1, 1}}}, // Q3_K (two runs; 2560x10240 at 4 columns 120 -> 70 us)};+// RTX PRO 5000 Blackwell (sm_120), 147-verify: read off mmvq_il_parity --bench --emit-table with the engine's shapes, the+// same 3% rule; it takes the interleaved kernels in more cells than the shared table (e.g. the Q5_K head at 2 columns+// 507 -> 367 us, IQ4_XS 2560x10240 at 3 columns 20.7 -> 14.9 us). Bitwise equal to native_mmvq.+constexpr IlRows kIlRows_120[] = {+ {23, {{0, 1, 2, 2, 2}, {0, 4, 4, 4, 4}, {0, 2, 2, 4, 2}}}, // IQ4_XS+ {12, {{0, 0, 2, 1, 1}, {0, 2, 4, 2, 2}, {0, 2, 2, 2, 2}}}, // Q4_K+ {13, {{0, 1, 2, 1, 1}, {0, 1, 4, 2, 4}, {0, 1, 2, 1, 2}}}, // Q5_K+ {14, {{0, 0, 2, 1, 1}, {0, 0, 4, 2, 4}, {0, 2, 2, 2, 2}}}, // Q6_K+ {11, {{0, 1, 2, 2, 2}, {0, 2, 4, 2, 2}, {0, 2, 2, 2, 2}}}, // Q3_K+};// Turing (sm_75): read off mmvq_il_parity --bench --emit-table on an RTX 2080 Ti 22 GB (SM 1650 MHz), the same 3% rule.// The plain multi-column kernels are bound by dp4a throughput on Turing from 2 columns on (IQ4_XS 2560x10240 at 4// columns 81.1 -> 37.4 us, the Q5_K head 1671 -> 1045 us); bitwise equal to native_mmvq. No one-column table here:@@ -3039,6 +3054,22 @@ constexpr IlRows kIlRows_75[] = {{14, {{0, 0, 1, 2, 1}, {0, 2, 2, 4, 4}, {0, 4, 4, 4, 4}}}, // Q6_K{11, {{1, 2, 4, 4, 4}, {1, 2, 4, 4, 4}, {1, 2, 2, 2, 2}}}, // Q3_K};+// 147-verify: 5-8 columns (a verify window of 5+ rows) by n_out class, per architecture; 0 = native_mmvq's kernels. Every+// choice is bitwise native_mmvq's output (mmvq_il_parity, T 1-8), so a table is only speed. Measured on an RTX PRO 5000+// Blackwell (sm_120) with mmvq_il_parity --bench --emit-table; other cards have no table yet and keep native_mmvq's+// kernels at 5+ columns. STRATA_MMVQ_IL_WIDE=0 turns the 5-8 column path off.+struct IlRows58 { int type; uint8_t r[4][5]; };+constexpr IlRows58 kIlRows58_120[] = { // 4 = 2 rows a warp here (launch_il_rows)+ {23, {{0, 4, 4, 4, 4}, {0, 4, 2, 4, 2}, {0, 4, 2, 4, 2}, {0, 4, 2, 2, 2}}}, // IQ4_XS+ {12, {{0, 2, 4, 4, 4}, {0, 4, 4, 4, 4}, {0, 2, 2, 2, 2}, {0, 4, 2, 4, 4}}}, // Q4_K+ {13, {{0, 2, 2, 2, 4}, {0, 4, 2, 4, 2}, {0, 2, 4, 2, 2}, {0, 2, 4, 2, 2}}}, // Q5_K+ {14, {{0, 2, 4, 4, 4}, {0, 4, 2, 4, 2}, {0, 2, 4, 2, 2}, {0, 4, 4, 4, 2}}}, // Q6_K+ {11, {{0, 2, 4, 2, 4}, {0, 2, 2, 4, 4}, {0, 2, 2, 2, 4}, {0, 4, 2, 2, 2}}}, // Q3_K+};+struct Il58Arch { int cc; const IlRows58* t; size_t n; };+constexpr Il58Arch kIl58Arch[] = {+ {120, kIlRows58_120, sizeof(kIlRows58_120) / sizeof(IlRows58)},+};// One column: rows a warp by n_out class, 0 = native_mmvq's single-column kernels (STRATA_MMVQ_IL_T1=0 turns this off).struct Il1Rows { int type; uint8_t r[5]; };constexpr Il1Rows kIl1Rows_86[] = {@@ -3068,6 +3099,7 @@ constexpr IlArch kIlArch[] = {{75, kIlRows_75, sizeof(kIlRows_75) / sizeof(IlRows)}, // Turing: measured on an RTX 2080 Ti{70, kIlRows_70, sizeof(kIlRows_70) / sizeof(IlRows)}, // Volta: PR 1401, measured on a V100-SXM2{86, kIlRows_86, sizeof(kIlRows_86) / sizeof(IlRows)}, // 143-dense: RTX A4000 with the real shapes (Q3_K added)+ {120, kIlRows_120, sizeof(kIlRows_120) / sizeof(IlRows)}, // 147-verify: RTX PRO 5000 Blackwell};// STRATA_MMVQ_IL_ROWS="type:ncols:r0,r1,r2,r3,r4;..." overrides single rows of the table for every card (type = ggml// type 23/12/13/14, ncols 2-4, r* = rows a warp 0/1/2/4 for the five n_out classes). Parsed once.@@ -3104,8 +3136,18 @@ const std::vector<IlEnvRow>& il_env_rows() {// The rows a warp for a card of compute capability `cc` (major * 10 + minor); 0: native_mmvq's kernels. Pure, so a test// can check every architecture's table without that card.int native_mmvq_il_rows_for(int cc, int type, int ncols, int n_out) {- if (ncols < 1 || ncols > 4) return 0;+ if (ncols < 1 || ncols > 8) return 0;const int cls = n_out < 2048 ? 0 : n_out < 4096 ? 1 : n_out < 8192 ? 2 : n_out < 12288 ? 3 : 4;+ if (ncols > 4) { // 147-verify: 5..8 columns, only where a card measured its table+ static const bool wide = il_env_on("STRATA_MMVQ_IL_WIDE");+ if (!wide) return 0;+ if (type == 11 && !il_q3k_on()) return 0;+ for (const Il58Arch& a : kIl58Arch)+ if (a.cc == cc)+ for (size_t i = 0; i < a.n; ++i)+ if (a.t[i].type == type) return a.t[i].r[ncols - 5][cls];+ return 0;+ }for (const IlEnvRow& e : il_env_rows())if (e.type == type && e.ncols == ncols) return e.r[cls];if (ncols == 1) { // 143-dense: one column (the plain q8_1 blocks, no interleaved copy); STRATA_MMVQ_IL_T1=0 turns it off@@ -3119,8 +3161,9 @@ int native_mmvq_il_rows_for(int cc, int type, int ncols, int n_out) {if (type == 11 && !il_q3k_on()) return 0; // STRATA_MMVQ_IL_Q3K=0: Q3_K keeps native_mmvq's kernelsconst IlRows* t = kIlRowsShared;size_t n = sizeof(kIlRowsShared) / sizeof(IlRows);+ static const bool own120 = il_env_on("STRATA_MMVQ_IL_120"); // 147-verify: =0 gives sm_120 the shared table againfor (const IlArch& a : kIlArch)- if (a.cc == cc && a.t != nullptr) { t = a.t; n = a.n; }+ if (a.cc == cc && a.t != nullptr && (cc != 120 || own120)) { t = a.t; n = a.n; }for (size_t i = 0; i < n; ++i)if (t[i].type == type) return t[i].r[ncols - 2][cls];return 0;@@ -3173,7 +3216,7 @@ bool il_arch_ok() {} // namespacebool native_mmvq_il_supported(int ggml_type, int ncols, int n_out) {- if (ncols < 1 || ncols > 4 || !g_multi_exact || !il_arch_ok()) return false;+ if (ncols < 1 || ncols > 8 || !g_multi_exact || !il_arch_ok()) return false;return (g_tune_rows ? g_tune_rows : il_rows(ggml_type, ncols, n_out)) != 0 &&(ggml_type == 23 || ggml_type == 12 || ggml_type == 13 || ggml_type == 14 || ggml_type == 11);}diff --git a/src/kernels/mmvq_il_parity.cpp b/src/kernels/mmvq_il_parity.cppindex 109b31c29..9e78015a2 100644--- a/src/kernels/mmvq_il_parity.cpp+++ b/src/kernels/mmvq_il_parity.cpp@@ -105,7 +105,8 @@ int run_case(const Case& c, bool bench, cudaStream_t s) {return 1;}int bad = 0;- for (int T = 1; T <= 4; ++T) {+ const int t_max = std::getenv("MMVQ_IL_TMAX") ? std::atoi(std::getenv("MMVQ_IL_TMAX")) : 8; // 147-verify: 5-8 columns+ for (int T = 1; T <= t_max; ++T) {std::vector<float> x((std::size_t) T * c.n_in);std::normal_distribution<float> nd(0.f, 1.f);for (auto& v : x) v = nd(rng);@@ -232,6 +233,20 @@ void emit_table() {}std::printf("}}, // %s\n", names[ti]);}+ std::printf("// 5-8 columns (kIlRows58_<cc>; rows 4 runs as 2 there):\n");+ for (int ti = 0; ti < 5; ++ti) {+ std::printf(" {%d, {", types[ti]);+ for (int nc = 5; nc <= 8; ++nc) {+ int prev = 0;+ std::printf("{");+ for (int cls = 0; cls < 5; ++cls) {+ prev = pick_cell(types[ti], nc, cls, prev);+ std::printf("%d%s", prev, cls < 4 ? ", " : "");+ }+ std::printf("}%s", nc < 8 ? ", " : "");+ }+ std::printf("}}, // %s\n", names[ti]);+ }std::printf("// one column (kIl1RowsShared):\n");for (int ti = 0; ti < 5; ++ti) {std::printf(" {%d, {", types[ti]);@@ -251,12 +266,13 @@ int check_tables() {for (int nc = 2; nc <= 4; ++nc)for (int n_out : {512, 2560, 6144, 10240, 12288, 248320}) {const int a = native_mmvq_il_rows_for(86, type, nc, n_out);- // sm_89 and sm_120 have no table of their own: they answer the shared table (what sm_86 answered before its- // own 143-dense table, which differs only in Q3_K, a type the shared table does not use)+ // sm_89 has no table of its own: it answers the shared table (what sm_86 answered before its own+ // 143-dense table, which differs only in Q3_K, a type the shared table does not use); sm_120 has its own+ // since 147-verifyconst int b = native_mmvq_il_rows_for(89, type, nc, n_out);- if (b != native_mmvq_il_rows_for(120, type, nc, n_out) || (type != 11 && b != a) || (type == 11 && b != 0)) {+ if ((type != 11 && b != a) || (type == 11 && b != 0)) {++bad;- std::printf("FAIL table: sm_89/sm_120 differ from sm_86 at type %d ncols %d n_out %d\n", type, nc, n_out);+ std::printf("FAIL table: sm_89 differs from sm_86 at type %d ncols %d n_out %d\n", type, nc, n_out);}if (a != 0 && a != 1 && a != 2 && a != 4) { ++bad; std::printf("FAIL table: rows %d\n", a); }}@@ -270,7 +286,8 @@ int check_tables() {++bad;std::printf("FAIL table: the sm_60 table is not the P100's\n");}- if (native_mmvq_il_rows_for(89, 99, 2, 2560) != 0 || native_mmvq_il_rows_for(89, 12, 5, 2560) != 0) {+ if (native_mmvq_il_rows_for(89, 99, 2, 2560) != 0 || native_mmvq_il_rows_for(89, 12, 5, 2560) != 0 ||+ native_mmvq_il_rows_for(86, 12, 5, 2560) != 0 || native_mmvq_il_rows_for(120, 12, 9, 2560) != 0) {++bad;std::printf("FAIL table: an unknown type or ncols must be 0\n");}@@ -289,6 +306,6 @@ int main(int argc, char** argv) {for (const Case& c : CASES)if (only == nullptr || std::strstr(c.name, only) != nullptr) bad += run_case(c, bench, s);if (emit) emit_table();- std::printf("%s\n", bad ? "mmvq_il_parity: FAIL" : "mmvq_il_parity: OK (every case, T 1-4, rows 1/2/4 and the table, bitwise)");+ std::printf("%s\n", bad ? "mmvq_il_parity: FAIL" : "mmvq_il_parity: OK (every case, T 1-8, rows 1/2/4 and the table, bitwise)");return bad ? 1 : 0;}