Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
native_mmvq: sm_89 MMVQ-IL tables opt-in with STRATA_MMVQ_IL_89=1 (#3453)
Niko12212299c8f70a28
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 6356e27b6..ed296bab1 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1111,6 +1111,9 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)target_include_directories(mmvq_il_parity PRIVATE ${_strata_gpu_include_directories})target_link_libraries(mmvq_il_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})add_test(NAME mmvq_il_parity COMMAND mmvq_il_parity)+ # sm_89's opt-in tables (#3453, STRATA_MMVQ_IL_89=1): the table check only (no case matches MMVQ_IL_ONLY)+ add_test(NAME mmvq_il_parity_89table COMMAND mmvq_il_parity)+ set_tests_properties(mmvq_il_parity_89table PROPERTIES ENVIRONMENT "STRATA_MMVQ_IL_89=1;MMVQ_IL_ONLY=no-such-case")# the wide dequant kernels (a thread per 8 values; sm_61's default), bitwise vs the per-group ones (GPU, synthetic,# no model; --bench adds old/new timings)add_executable(dequant_wide_parity src/kernels/dequant_wide_parity.cpp)diff --git a/docs/MMVQ_IL_TABLE.md b/docs/MMVQ_IL_TABLE.mdindex c49296258..8974c3be9 100644--- a/docs/MMVQ_IL_TABLE.md+++ b/docs/MMVQ_IL_TABLE.md@@ -15,7 +15,7 @@ Which table a card gets:|---|---|| sm_86 (Ampere consumer) and sm_120 cards with fewer than 80 SMs (RTX 5070 and smaller) | `kIlRowsShared`, measured on an RTX 3060 and an RTX 5070 together (a cell is non-zero only if the worse of the two cards still gains 3%) || sm_120 cards with 80 or more SMs (RTX PRO 5000 / 6000 Blackwell, RTX 5080 / 5090) | `kIlRows_120`, measured on an RTX PRO 5000 Blackwell (110 SMs); `STRATA_MMVQ_IL_120=0` gives them the shared table |-| sm_89 (Ada) | `kIlRows_89`: the cells where an RTX 4070 (46 SMs) and an RTX 4080 (48 SMs) community run gave the same rows count, else 0 (the plain kernel); one column: Q3_K only. No 5-8 column table: the 4080's changed in 10-14 of 20 entries between runs of the same card, so it is not taken until it reproduces. No end-to-end decode change was seen on a 4090 or a 4080 (issue 3453), so this is a microbenchmark gain only. |+| sm_89 (Ada) | the shared table (`kIlRowsShared`) unless `STRATA_MMVQ_IL_89=1`, which switches on `kIlRows_89`: the cells where an RTX 4070 (46 SMs) and an RTX 4080 (48 SMs) community run gave the same rows count, else 0 (the plain kernel); one column: Q3_K only. It is opt-in because no sm_89 card has run `mmvq_il_parity` on this code, and the issue saw no end-to-end decode change on a 4090 or a 4080 (issue 3453), so the gain is a microbenchmark one. No 5-8 column table: the 4080's changed in 10-14 of 20 entries between runs of the same card. To flip the default, run `mmvq_il_parity` on an Ada card (it must end with `OK`) and an interleaved decode A/B with the switch on and off. || every other sm_80+ card | `kIlRowsShared` || sm_75 and older | the path is off; the plain kernels run || a card listed in `kIlArch` | its own table |diff --git a/src/kernels/cuda/native_mmvq.cu b/src/kernels/cuda/native_mmvq.cuindex c4b119425..f0c3913ef 100644--- a/src/kernels/cuda/native_mmvq.cu+++ b/src/kernels/cuda/native_mmvq.cu@@ -3037,7 +3037,7 @@ constexpr IlRows kIlRowsShared[] = { // sm_86 + sm_120, and the fallback for e{14, {{0, 0, 1, 1, 0}, {0, 0, 1, 1, 2}, {0, 0, 1, 1, 1}}}, // Q6_K};// Per-architecture tables (P10). A card whose compute capability (major * 10 + minor) is listed here uses its own table;-// every other sm_80+ card uses kIlRowsShared. sm_89 (Ada) has its own table (kIlRows_89, below). To add one, run+// every other sm_80+ card uses kIlRowsShared. sm_89 (Ada) has an opt-in table (kIlRows_89, below; STRATA_MMVQ_IL_89=1). To add one, run// `mmvq_il_parity --bench --emit-table` on the card (docs/MMVQ_IL_TABLE.md), paste the printed block as a// `constexpr IlRows kIlRows_<cc>[]` and add {<cc>, kIlRows_<cc>, sizeof(kIlRows_<cc>) / sizeof(IlRows)} below. Each choice// is bitwise the same output, so a table is only speed. Without a rebuild: STRATA_MMVQ_IL_ROWS (see il_env_rows).@@ -3158,6 +3158,10 @@ bool il_env_on(const char* name) {}bool il_t1_on() { static const bool on = il_env_on("STRATA_MMVQ_IL_T1"); return on; }bool il_q3k_on() { static const bool on = il_env_on("STRATA_MMVQ_IL_Q3K"); return on; }+// #3453: sm_89's own tables (kIlRows_89 / kIl1Rows_89) are opt-in with STRATA_MMVQ_IL_89=1 until an sm_89 card has run+// mmvq_il_parity and an end-to-end A/B (docs/MMVQ_IL_TABLE.md). Unset, an Ada card answers what it answered before: the+// shared table, no one-column row. Cached once; the lookup runs on the host once a launch.+bool il_89_opt_in() { static const bool on = [] { const char* v = std::getenv("STRATA_MMVQ_IL_89"); return v != nullptr && v[0] == '1'; }(); return on; }struct IlArch { int cc; const IlRows* t; size_t n; };constexpr IlArch kIlArch[] = {{60, kIlRows_60, sizeof(kIlRows_60) / sizeof(IlRows)}, // Pascal GP100: measured on a Tesla P100@@ -3222,7 +3226,7 @@ int native_mmvq_il_rows_for(int cc, int type, int ncols, int n_out) {if (ncols == 1) { // 143-dense: one column (the plain q8_1 blocks, no interleaved copy); STRATA_MMVQ_IL_T1=0 turns it offif (!il_t1_on()) return 0;for (const Il1Arch& a : kIl1Arch)- if (a.cc == cc)+ if (a.cc == cc && (cc != 89 || il_89_opt_in()))for (size_t i = 0; i < a.n; ++i)if (a.t[i].type == type) return a.t[i].r[cls];return 0;@@ -3232,7 +3236,7 @@ int native_mmvq_il_rows_for(int cc, int type, int ncols, int n_out) {size_t n = sizeof(kIlRowsShared) / sizeof(IlRows);static const bool own120 = il_env_on("STRATA_MMVQ_IL_120"); // 147-verify: =0 gives a big 12.0 card the shared table againfor (const IlArch& a : kIlArch)- if (a.cc == cc && a.t != nullptr && (cc != 1201 || own120)) { t = a.t; n = a.n; }+ if (a.cc == cc && a.t != nullptr && (cc != 1201 || own120) && (cc != 89 || il_89_opt_in())) { t = a.t; n = a.n; }for (size_t i = 0; i < n; ++i)if (t[i].type == type) return t[i].r[ncols - 2][cls];return 0;diff --git a/src/kernels/mmvq_il_parity.cpp b/src/kernels/mmvq_il_parity.cppindex 05225d9f2..9e9cd1654 100644--- a/src/kernels/mmvq_il_parity.cpp+++ b/src/kernels/mmvq_il_parity.cpp@@ -272,6 +272,8 @@ const int kAda4080[5][3][5] = {{{0, 0, 1, 1, 1}, {0, 1, 1, 1, 1}, {0, 1, 1, 1, 1}}};int check_tables() {int bad = 0;+ const char* e89 = std::getenv("STRATA_MMVQ_IL_89");+ const bool opt89 = e89 != nullptr && e89[0] == '1'; // sm_89's own tables are opt-in (#3453)for (int type : {23, 12, 13, 14, 11})for (int nc = 2; nc <= 4; ++nc)for (int n_out : {512, 2560, 6144, 10240, 12288, 248320}) {@@ -281,11 +283,14 @@ int check_tables() {const int cls = n_out < 2048 ? 0 : n_out < 4096 ? 1 : n_out < 8192 ? 2 : n_out < 12288 ? 3 : 4;const int ti = type == 23 ? 0 : type == 12 ? 1 : type == 13 ? 2 : type == 14 ? 3 : 4;const int b = native_mmvq_il_rows_for(89, type, nc, n_out);- const int want = kAda4070[ti][nc - 2][cls] == kAda4080[ti][nc - 2][cls] ? kAda4070[ti][nc - 2][cls] : 0;+ // STRATA_MMVQ_IL_89=1: the agreement; unset: what sm_89 answered before #3453 (the shared table, which is+ // sm_86's except Q3_K, a type the shared table has no row for)+ const int want = opt89 ? (kAda4070[ti][nc - 2][cls] == kAda4080[ti][nc - 2][cls] ? kAda4070[ti][nc - 2][cls] : 0)+ : (type == 11 ? 0 : a);if (b != want) {++bad;- std::printf("FAIL table: sm_89 is %d, the 4070/4080 agreement says %d at type %d ncols %d n_out %d\n", b, want,- type, nc, n_out);+ std::printf("FAIL table: sm_89 is %d, %s says %d at type %d ncols %d n_out %d\n", b,+ opt89 ? "the 4070/4080 agreement" : "the shared table", want, type, nc, n_out);}}// Volta has its own table (PR 1401): Q6_K at 2 columns, 2560 rows -> 1 row a warp; the Ampere table says 0 there@@ -298,8 +303,9 @@ int check_tables() {++bad;std::printf("FAIL table: the sm_60 table is not the P100's\n");}- // sm_89 one column: only Q3_K (the 4070 and the 4080 agree); wider than 4: the plain kernels- if (native_mmvq_il_rows_for(89, 11, 1, 2560) != 0 || native_mmvq_il_rows_for(89, 11, 1, 6144) != 1 || native_mmvq_il_rows_for(89, 12, 1, 6144) != 0) {+ // sm_89 one column: only Q3_K with STRATA_MMVQ_IL_89=1 (the 4070 and the 4080 agree), else none; wider than 4: the plain kernels+ if (native_mmvq_il_rows_for(89, 11, 1, 2560) != 0 || native_mmvq_il_rows_for(89, 11, 1, 6144) != (opt89 ? 1 : 0) ||+ native_mmvq_il_rows_for(89, 12, 1, 6144) != 0) {++bad;std::printf("FAIL table: the sm_89 one-column row is not Q3_K only\n");}