Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

mmvq_il: 5-8 column interleaved kernels and the sm_120 tables (bitwise)

Niko122175fabbd94152
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index 427c712ac..52a96be44 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -765,7 +765,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
lo_ = b.take<float>(T * (uint64_t) g.hc_lr); rs_ = b.take<float>(T * HC); xn_ = b.take<float>(T * HC * N);
xq_ = b.take<uint8_t>(strata::kernels::native_q8_1_bytes(max_in, (int) T));
sh_xq_ = b.take<uint8_t>(strata::kernels::native_q8_1_bytes(max_in, (int) T));
- xil_ = b.take<uint8_t>(strata::kernels::native_q8_1_il_bytes(max_in, 4));
+ xil_ = b.take<uint8_t>(strata::kernels::native_q8_1_il_bytes(max_in, std::max(4, (int) T))); // 147-verify: 5-8 columns too
qkv_L_ = b.take<float>(nG * T * C); h_L_ = b.take<float>(nG * T * C);
gate_L_ = b.take<float>(nG * T * HV); beta_L_ = b.take<float>(nG * T * HV);
z_ = b.take<float>(T * ZV); y_ = b.take<float>(T * ZV); y_dummy_ = b.take<float>(T * ZV);
diff --git a/src/kernels/cuda/native_mmvq.cu b/src/kernels/cuda/native_mmvq.cu
index 2e3af20cd..ec0da97bb 100644
--- a/src/kernels/cuda/native_mmvq.cu
+++ b/src/kernels/cuda/native_mmvq.cu
@@ -1730,12 +1730,17 @@ void launch_il(const void* weights, const void* x_il, float* y, int n_in, int n_
#define STRATA_IL_CASE(N) case N: native_mmvq_il_kernel<I, N, R><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); break;
STRATA_IL_CASE(2) STRATA_IL_CASE(3) STRATA_IL_CASE(4)
#undef STRATA_IL_CASE
- default: throw std::invalid_argument("native_mmvq_il requires 1 <= ncols <= 4");
+ // 147-verify: 5..8 columns (the cp = 8 copy) at 1 or 2 rows a warp; launch_il_rows maps 4 rows to 2 there
+#define STRATA_IL_CASE(N) case N: if constexpr (R <= 2) native_mmvq_il_kernel<I, N, R><<<blocks, 4 * WARP, 0, s>>>(w, xq, xd, y, n_in, n_out); break;
+ STRATA_IL_CASE(5) STRATA_IL_CASE(6) STRATA_IL_CASE(7) STRATA_IL_CASE(8)
+#undef STRATA_IL_CASE
+ default: throw std::invalid_argument("native_mmvq_il requires 1 <= ncols <= 8");
}
}
template <typename I>
void launch_il_rows(int r, const void* weights, const void* x_il, float* y, int n_in, int n_out, int ncols, cudaStream_t s) {
+ if (ncols > 4 && r == 4) r = 2; // 5..8 columns: 1 or 2 rows a warp
switch (r) {
case 1: launch_il<I, 1>(weights, x_il, y, n_in, n_out, ncols, s); break;
case 4: launch_il<I, 4>(weights, x_il, y, n_in, n_out, ncols, s); break;
@@ -2956,7 +2961,7 @@ void native_quantize_q8_1_il(const float* x, void* x_q8_1, void* x_il, int n_in,
void native_q8_1_interleave(const void* x_q8_1, void* x_il, int n_in, int ncols, void* stream) {
validate_shape(n_in, ncols);
- if (ncols < 2 || ncols > 4) throw std::invalid_argument("native_q8_1_interleave requires 2 <= ncols <= 4");
+ if (ncols < 2 || ncols > 8) throw std::invalid_argument("native_q8_1_interleave requires 2 <= ncols <= 8");
validate_pointer(x_q8_1);
validate_pointer(x_il);
validate_stream(stream);
@@ -3028,6 +3033,16 @@ constexpr IlRows kIlRows_86[] = {
{14, {{0, 0, 1, 1, 0}, {0, 0, 1, 1, 2}, {0, 0, 1, 1, 1}}}, // Q6_K
{11, {{1, 2, 2, 2, 2}, {1, 1, 1, 2, 2}, {1, 1, 1, 1, 1}}}, // Q3_K (two runs; 2560x10240 at 4 columns 120 -> 70 us)
};
+// RTX PRO 5000 Blackwell (sm_120), 147-verify: read off mmvq_il_parity --bench --emit-table with the engine's shapes, the
+// same 3% rule; it takes the interleaved kernels in more cells than the shared table (e.g. the Q5_K head at 2 columns
+// 507 -> 367 us, IQ4_XS 2560x10240 at 3 columns 20.7 -> 14.9 us). Bitwise equal to native_mmvq.
+constexpr IlRows kIlRows_120[] = {
+ {23, {{0, 1, 2, 2, 2}, {0, 4, 4, 4, 4}, {0, 2, 2, 4, 2}}}, // IQ4_XS
+ {12, {{0, 0, 2, 1, 1}, {0, 2, 4, 2, 2}, {0, 2, 2, 2, 2}}}, // Q4_K
+ {13, {{0, 1, 2, 1, 1}, {0, 1, 4, 2, 4}, {0, 1, 2, 1, 2}}}, // Q5_K
+ {14, {{0, 0, 2, 1, 1}, {0, 0, 4, 2, 4}, {0, 2, 2, 2, 2}}}, // Q6_K
+ {11, {{0, 1, 2, 2, 2}, {0, 2, 4, 2, 2}, {0, 2, 2, 2, 2}}}, // Q3_K
+};
// Turing (sm_75): read off mmvq_il_parity --bench --emit-table on an RTX 2080 Ti 22 GB (SM 1650 MHz), the same 3% rule.
// The plain multi-column kernels are bound by dp4a throughput on Turing from 2 columns on (IQ4_XS 2560x10240 at 4
// columns 81.1 -> 37.4 us, the Q5_K head 1671 -> 1045 us); bitwise equal to native_mmvq. No one-column table here:
@@ -3039,6 +3054,22 @@ constexpr IlRows kIlRows_75[] = {
{14, {{0, 0, 1, 2, 1}, {0, 2, 2, 4, 4}, {0, 4, 4, 4, 4}}}, // Q6_K
{11, {{1, 2, 4, 4, 4}, {1, 2, 4, 4, 4}, {1, 2, 2, 2, 2}}}, // Q3_K
};
+// 147-verify: 5-8 columns (a verify window of 5+ rows) by n_out class, per architecture; 0 = native_mmvq's kernels. Every
+// choice is bitwise native_mmvq's output (mmvq_il_parity, T 1-8), so a table is only speed. Measured on an RTX PRO 5000
+// Blackwell (sm_120) with mmvq_il_parity --bench --emit-table; other cards have no table yet and keep native_mmvq's
+// kernels at 5+ columns. STRATA_MMVQ_IL_WIDE=0 turns the 5-8 column path off.
+struct IlRows58 { int type; uint8_t r[4][5]; };
+constexpr IlRows58 kIlRows58_120[] = { // 4 = 2 rows a warp here (launch_il_rows)
+ {23, {{0, 4, 4, 4, 4}, {0, 4, 2, 4, 2}, {0, 4, 2, 4, 2}, {0, 4, 2, 2, 2}}}, // IQ4_XS
+ {12, {{0, 2, 4, 4, 4}, {0, 4, 4, 4, 4}, {0, 2, 2, 2, 2}, {0, 4, 2, 4, 4}}}, // Q4_K
+ {13, {{0, 2, 2, 2, 4}, {0, 4, 2, 4, 2}, {0, 2, 4, 2, 2}, {0, 2, 4, 2, 2}}}, // Q5_K
+ {14, {{0, 2, 4, 4, 4}, {0, 4, 2, 4, 2}, {0, 2, 4, 2, 2}, {0, 4, 4, 4, 2}}}, // Q6_K
+ {11, {{0, 2, 4, 2, 4}, {0, 2, 2, 4, 4}, {0, 2, 2, 2, 4}, {0, 4, 2, 2, 2}}}, // Q3_K
+};
+struct Il58Arch { int cc; const IlRows58* t; size_t n; };
+constexpr Il58Arch kIl58Arch[] = {
+ {120, kIlRows58_120, sizeof(kIlRows58_120) / sizeof(IlRows58)},
+};
// One column: rows a warp by n_out class, 0 = native_mmvq's single-column kernels (STRATA_MMVQ_IL_T1=0 turns this off).
struct Il1Rows { int type; uint8_t r[5]; };
constexpr Il1Rows kIl1Rows_86[] = {
@@ -3068,6 +3099,7 @@ constexpr IlArch kIlArch[] = {
{75, kIlRows_75, sizeof(kIlRows_75) / sizeof(IlRows)}, // Turing: measured on an RTX 2080 Ti
{70, kIlRows_70, sizeof(kIlRows_70) / sizeof(IlRows)}, // Volta: PR 1401, measured on a V100-SXM2
{86, kIlRows_86, sizeof(kIlRows_86) / sizeof(IlRows)}, // 143-dense: RTX A4000 with the real shapes (Q3_K added)
+ {120, kIlRows_120, sizeof(kIlRows_120) / sizeof(IlRows)}, // 147-verify: RTX PRO 5000 Blackwell
};
// STRATA_MMVQ_IL_ROWS="type:ncols:r0,r1,r2,r3,r4;..." overrides single rows of the table for every card (type = ggml
// type 23/12/13/14, ncols 2-4, r* = rows a warp 0/1/2/4 for the five n_out classes). Parsed once.
@@ -3104,8 +3136,18 @@ const std::vector<IlEnvRow>& il_env_rows() {
// The rows a warp for a card of compute capability `cc` (major * 10 + minor); 0: native_mmvq's kernels. Pure, so a test
// can check every architecture's table without that card.
int native_mmvq_il_rows_for(int cc, int type, int ncols, int n_out) {
- if (ncols < 1 || ncols > 4) return 0;
+ if (ncols < 1 || ncols > 8) return 0;
const int cls = n_out < 2048 ? 0 : n_out < 4096 ? 1 : n_out < 8192 ? 2 : n_out < 12288 ? 3 : 4;
+ if (ncols > 4) { // 147-verify: 5..8 columns, only where a card measured its table
+ static const bool wide = il_env_on("STRATA_MMVQ_IL_WIDE");
+ if (!wide) return 0;
+ if (type == 11 && !il_q3k_on()) return 0;
+ for (const Il58Arch& a : kIl58Arch)
+ if (a.cc == cc)
+ for (size_t i = 0; i < a.n; ++i)
+ if (a.t[i].type == type) return a.t[i].r[ncols - 5][cls];
+ return 0;
+ }
for (const IlEnvRow& e : il_env_rows())
if (e.type == type && e.ncols == ncols) return e.r[cls];
if (ncols == 1) { // 143-dense: one column (the plain q8_1 blocks, no interleaved copy); STRATA_MMVQ_IL_T1=0 turns it off
@@ -3119,8 +3161,9 @@ int native_mmvq_il_rows_for(int cc, int type, int ncols, int n_out) {
if (type == 11 && !il_q3k_on()) return 0; // STRATA_MMVQ_IL_Q3K=0: Q3_K keeps native_mmvq's kernels
const IlRows* t = kIlRowsShared;
size_t n = sizeof(kIlRowsShared) / sizeof(IlRows);
+ static const bool own120 = il_env_on("STRATA_MMVQ_IL_120"); // 147-verify: =0 gives sm_120 the shared table again
for (const IlArch& a : kIlArch)
- if (a.cc == cc && a.t != nullptr) { t = a.t; n = a.n; }
+ if (a.cc == cc && a.t != nullptr && (cc != 120 || own120)) { t = a.t; n = a.n; }
for (size_t i = 0; i < n; ++i)
if (t[i].type == type) return t[i].r[ncols - 2][cls];
return 0;
@@ -3173,7 +3216,7 @@ bool il_arch_ok() {
} // namespace
bool native_mmvq_il_supported(int ggml_type, int ncols, int n_out) {
- if (ncols < 1 || ncols > 4 || !g_multi_exact || !il_arch_ok()) return false;
+ if (ncols < 1 || ncols > 8 || !g_multi_exact || !il_arch_ok()) return false;
return (g_tune_rows ? g_tune_rows : il_rows(ggml_type, ncols, n_out)) != 0 &&
(ggml_type == 23 || ggml_type == 12 || ggml_type == 13 || ggml_type == 14 || ggml_type == 11);
}
diff --git a/src/kernels/mmvq_il_parity.cpp b/src/kernels/mmvq_il_parity.cpp
index 109b31c29..9e78015a2 100644
--- a/src/kernels/mmvq_il_parity.cpp
+++ b/src/kernels/mmvq_il_parity.cpp
@@ -105,7 +105,8 @@ int run_case(const Case& c, bool bench, cudaStream_t s) {
return 1;
}
int bad = 0;
- for (int T = 1; T <= 4; ++T) {
+ const int t_max = std::getenv("MMVQ_IL_TMAX") ? std::atoi(std::getenv("MMVQ_IL_TMAX")) : 8; // 147-verify: 5-8 columns
+ for (int T = 1; T <= t_max; ++T) {
std::vector<float> x((std::size_t) T * c.n_in);
std::normal_distribution<float> nd(0.f, 1.f);
for (auto& v : x) v = nd(rng);
@@ -232,6 +233,20 @@ void emit_table() {
}
std::printf("}}, // %s\n", names[ti]);
}
+ std::printf("// 5-8 columns (kIlRows58_<cc>; rows 4 runs as 2 there):\n");
+ for (int ti = 0; ti < 5; ++ti) {
+ std::printf(" {%d, {", types[ti]);
+ for (int nc = 5; nc <= 8; ++nc) {
+ int prev = 0;
+ std::printf("{");
+ for (int cls = 0; cls < 5; ++cls) {
+ prev = pick_cell(types[ti], nc, cls, prev);
+ std::printf("%d%s", prev, cls < 4 ? ", " : "");
+ }
+ std::printf("}%s", nc < 8 ? ", " : "");
+ }
+ std::printf("}}, // %s\n", names[ti]);
+ }
std::printf("// one column (kIl1RowsShared):\n");
for (int ti = 0; ti < 5; ++ti) {
std::printf(" {%d, {", types[ti]);
@@ -251,12 +266,13 @@ int check_tables() {
for (int nc = 2; nc <= 4; ++nc)
for (int n_out : {512, 2560, 6144, 10240, 12288, 248320}) {
const int a = native_mmvq_il_rows_for(86, type, nc, n_out);
- // sm_89 and sm_120 have no table of their own: they answer the shared table (what sm_86 answered before its
- // own 143-dense table, which differs only in Q3_K, a type the shared table does not use)
+ // sm_89 has no table of its own: it answers the shared table (what sm_86 answered before its own
+ // 143-dense table, which differs only in Q3_K, a type the shared table does not use); sm_120 has its own
+ // since 147-verify
const int b = native_mmvq_il_rows_for(89, type, nc, n_out);
- if (b != native_mmvq_il_rows_for(120, type, nc, n_out) || (type != 11 && b != a) || (type == 11 && b != 0)) {
+ if ((type != 11 && b != a) || (type == 11 && b != 0)) {
++bad;
- std::printf("FAIL table: sm_89/sm_120 differ from sm_86 at type %d ncols %d n_out %d\n", type, nc, n_out);
+ std::printf("FAIL table: sm_89 differs from sm_86 at type %d ncols %d n_out %d\n", type, nc, n_out);
}
if (a != 0 && a != 1 && a != 2 && a != 4) { ++bad; std::printf("FAIL table: rows %d\n", a); }
}
@@ -270,7 +286,8 @@ int check_tables() {
++bad;
std::printf("FAIL table: the sm_60 table is not the P100's\n");
}
- if (native_mmvq_il_rows_for(89, 99, 2, 2560) != 0 || native_mmvq_il_rows_for(89, 12, 5, 2560) != 0) {
+ if (native_mmvq_il_rows_for(89, 99, 2, 2560) != 0 || native_mmvq_il_rows_for(89, 12, 5, 2560) != 0 ||
+ native_mmvq_il_rows_for(86, 12, 5, 2560) != 0 || native_mmvq_il_rows_for(120, 12, 9, 2560) != 0) {
++bad;
std::printf("FAIL table: an unknown type or ncols must be 0\n");
}
@@ -289,6 +306,6 @@ int main(int argc, char** argv) {
for (const Case& c : CASES)
if (only == nullptr || std::strstr(c.name, only) != nullptr) bad += run_case(c, bench, s);
if (emit) emit_table();
- std::printf("%s\n", bad ? "mmvq_il_parity: FAIL" : "mmvq_il_parity: OK (every case, T 1-4, rows 1/2/4 and the table, bitwise)");
+ std::printf("%s\n", bad ? "mmvq_il_parity: FAIL" : "mmvq_il_parity: OK (every case, T 1-8, rows 1/2/4 and the table, bitwise)");
return bad ? 1 : 0;
}