Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

native_mmvq: sm_89 MMVQ-IL tables opt-in with STRATA_MMVQ_IL_89=1 (#3453)

Niko12212299c8f70a28
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6356e27b6..ed296bab1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1111,6 +1111,9 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
target_include_directories(mmvq_il_parity PRIVATE ${_strata_gpu_include_directories})
target_link_libraries(mmvq_il_parity PRIVATE strata_kernels ${_strata_gpu_runtime_target})
add_test(NAME mmvq_il_parity COMMAND mmvq_il_parity)
+ # sm_89's opt-in tables (#3453, STRATA_MMVQ_IL_89=1): the table check only (no case matches MMVQ_IL_ONLY)
+ add_test(NAME mmvq_il_parity_89table COMMAND mmvq_il_parity)
+ set_tests_properties(mmvq_il_parity_89table PROPERTIES ENVIRONMENT "STRATA_MMVQ_IL_89=1;MMVQ_IL_ONLY=no-such-case")
# the wide dequant kernels (a thread per 8 values; sm_61's default), bitwise vs the per-group ones (GPU, synthetic,
# no model; --bench adds old/new timings)
add_executable(dequant_wide_parity src/kernels/dequant_wide_parity.cpp)
diff --git a/docs/MMVQ_IL_TABLE.md b/docs/MMVQ_IL_TABLE.md
index c49296258..8974c3be9 100644
--- a/docs/MMVQ_IL_TABLE.md
+++ b/docs/MMVQ_IL_TABLE.md
@@ -15,7 +15,7 @@ Which table a card gets:
|---|---|
| sm_86 (Ampere consumer) and sm_120 cards with fewer than 80 SMs (RTX 5070 and smaller) | `kIlRowsShared`, measured on an RTX 3060 and an RTX 5070 together (a cell is non-zero only if the worse of the two cards still gains 3%) |
| sm_120 cards with 80 or more SMs (RTX PRO 5000 / 6000 Blackwell, RTX 5080 / 5090) | `kIlRows_120`, measured on an RTX PRO 5000 Blackwell (110 SMs); `STRATA_MMVQ_IL_120=0` gives them the shared table |
-| sm_89 (Ada) | `kIlRows_89`: the cells where an RTX 4070 (46 SMs) and an RTX 4080 (48 SMs) community run gave the same rows count, else 0 (the plain kernel); one column: Q3_K only. No 5-8 column table: the 4080's changed in 10-14 of 20 entries between runs of the same card, so it is not taken until it reproduces. No end-to-end decode change was seen on a 4090 or a 4080 (issue 3453), so this is a microbenchmark gain only. |
+| sm_89 (Ada) | the shared table (`kIlRowsShared`) unless `STRATA_MMVQ_IL_89=1`, which switches on `kIlRows_89`: the cells where an RTX 4070 (46 SMs) and an RTX 4080 (48 SMs) community run gave the same rows count, else 0 (the plain kernel); one column: Q3_K only. It is opt-in because no sm_89 card has run `mmvq_il_parity` on this code, and the issue saw no end-to-end decode change on a 4090 or a 4080 (issue 3453), so the gain is a microbenchmark one. No 5-8 column table: the 4080's changed in 10-14 of 20 entries between runs of the same card. To flip the default, run `mmvq_il_parity` on an Ada card (it must end with `OK`) and an interleaved decode A/B with the switch on and off. |
| every other sm_80+ card | `kIlRowsShared` |
| sm_75 and older | the path is off; the plain kernels run |
| a card listed in `kIlArch` | its own table |
diff --git a/src/kernels/cuda/native_mmvq.cu b/src/kernels/cuda/native_mmvq.cu
index c4b119425..f0c3913ef 100644
--- a/src/kernels/cuda/native_mmvq.cu
+++ b/src/kernels/cuda/native_mmvq.cu
@@ -3037,7 +3037,7 @@ constexpr IlRows kIlRowsShared[] = { // sm_86 + sm_120, and the fallback for e
{14, {{0, 0, 1, 1, 0}, {0, 0, 1, 1, 2}, {0, 0, 1, 1, 1}}}, // Q6_K
};
// Per-architecture tables (P10). A card whose compute capability (major * 10 + minor) is listed here uses its own table;
-// every other sm_80+ card uses kIlRowsShared. sm_89 (Ada) has its own table (kIlRows_89, below). To add one, run
+// every other sm_80+ card uses kIlRowsShared. sm_89 (Ada) has an opt-in table (kIlRows_89, below; STRATA_MMVQ_IL_89=1). To add one, run
// `mmvq_il_parity --bench --emit-table` on the card (docs/MMVQ_IL_TABLE.md), paste the printed block as a
// `constexpr IlRows kIlRows_<cc>[]` and add {<cc>, kIlRows_<cc>, sizeof(kIlRows_<cc>) / sizeof(IlRows)} below. Each choice
// is bitwise the same output, so a table is only speed. Without a rebuild: STRATA_MMVQ_IL_ROWS (see il_env_rows).
@@ -3158,6 +3158,10 @@ bool il_env_on(const char* name) {
}
bool il_t1_on() { static const bool on = il_env_on("STRATA_MMVQ_IL_T1"); return on; }
bool il_q3k_on() { static const bool on = il_env_on("STRATA_MMVQ_IL_Q3K"); return on; }
+// #3453: sm_89's own tables (kIlRows_89 / kIl1Rows_89) are opt-in with STRATA_MMVQ_IL_89=1 until an sm_89 card has run
+// mmvq_il_parity and an end-to-end A/B (docs/MMVQ_IL_TABLE.md). Unset, an Ada card answers what it answered before: the
+// shared table, no one-column row. Cached once; the lookup runs on the host once a launch.
+bool il_89_opt_in() { static const bool on = [] { const char* v = std::getenv("STRATA_MMVQ_IL_89"); return v != nullptr && v[0] == '1'; }(); return on; }
struct IlArch { int cc; const IlRows* t; size_t n; };
constexpr IlArch kIlArch[] = {
{60, kIlRows_60, sizeof(kIlRows_60) / sizeof(IlRows)}, // Pascal GP100: measured on a Tesla P100
@@ -3222,7 +3226,7 @@ int native_mmvq_il_rows_for(int cc, int type, int ncols, int n_out) {
if (ncols == 1) { // 143-dense: one column (the plain q8_1 blocks, no interleaved copy); STRATA_MMVQ_IL_T1=0 turns it off
if (!il_t1_on()) return 0;
for (const Il1Arch& a : kIl1Arch)
- if (a.cc == cc)
+ if (a.cc == cc && (cc != 89 || il_89_opt_in()))
for (size_t i = 0; i < a.n; ++i)
if (a.t[i].type == type) return a.t[i].r[cls];
return 0;
@@ -3232,7 +3236,7 @@ int native_mmvq_il_rows_for(int cc, int type, int ncols, int n_out) {
size_t n = sizeof(kIlRowsShared) / sizeof(IlRows);
static const bool own120 = il_env_on("STRATA_MMVQ_IL_120"); // 147-verify: =0 gives a big 12.0 card the shared table again
for (const IlArch& a : kIlArch)
- if (a.cc == cc && a.t != nullptr && (cc != 1201 || own120)) { t = a.t; n = a.n; }
+ if (a.cc == cc && a.t != nullptr && (cc != 1201 || own120) && (cc != 89 || il_89_opt_in())) { t = a.t; n = a.n; }
for (size_t i = 0; i < n; ++i)
if (t[i].type == type) return t[i].r[ncols - 2][cls];
return 0;
diff --git a/src/kernels/mmvq_il_parity.cpp b/src/kernels/mmvq_il_parity.cpp
index 05225d9f2..9e9cd1654 100644
--- a/src/kernels/mmvq_il_parity.cpp
+++ b/src/kernels/mmvq_il_parity.cpp
@@ -272,6 +272,8 @@ const int kAda4080[5][3][5] = {
{{0, 0, 1, 1, 1}, {0, 1, 1, 1, 1}, {0, 1, 1, 1, 1}}};
int check_tables() {
int bad = 0;
+ const char* e89 = std::getenv("STRATA_MMVQ_IL_89");
+ const bool opt89 = e89 != nullptr && e89[0] == '1'; // sm_89's own tables are opt-in (#3453)
for (int type : {23, 12, 13, 14, 11})
for (int nc = 2; nc <= 4; ++nc)
for (int n_out : {512, 2560, 6144, 10240, 12288, 248320}) {
@@ -281,11 +283,14 @@ int check_tables() {
const int cls = n_out < 2048 ? 0 : n_out < 4096 ? 1 : n_out < 8192 ? 2 : n_out < 12288 ? 3 : 4;
const int ti = type == 23 ? 0 : type == 12 ? 1 : type == 13 ? 2 : type == 14 ? 3 : 4;
const int b = native_mmvq_il_rows_for(89, type, nc, n_out);
- const int want = kAda4070[ti][nc - 2][cls] == kAda4080[ti][nc - 2][cls] ? kAda4070[ti][nc - 2][cls] : 0;
+ // STRATA_MMVQ_IL_89=1: the agreement; unset: what sm_89 answered before #3453 (the shared table, which is
+ // sm_86's except Q3_K, a type the shared table has no row for)
+ const int want = opt89 ? (kAda4070[ti][nc - 2][cls] == kAda4080[ti][nc - 2][cls] ? kAda4070[ti][nc - 2][cls] : 0)
+ : (type == 11 ? 0 : a);
if (b != want) {
++bad;
- std::printf("FAIL table: sm_89 is %d, the 4070/4080 agreement says %d at type %d ncols %d n_out %d\n", b, want,
- type, nc, n_out);
+ std::printf("FAIL table: sm_89 is %d, %s says %d at type %d ncols %d n_out %d\n", b,
+ opt89 ? "the 4070/4080 agreement" : "the shared table", want, type, nc, n_out);
}
}
// Volta has its own table (PR 1401): Q6_K at 2 columns, 2560 rows -> 1 row a warp; the Ampere table says 0 there
@@ -298,8 +303,9 @@ int check_tables() {
++bad;
std::printf("FAIL table: the sm_60 table is not the P100's\n");
}
- // sm_89 one column: only Q3_K (the 4070 and the 4080 agree); wider than 4: the plain kernels
- if (native_mmvq_il_rows_for(89, 11, 1, 2560) != 0 || native_mmvq_il_rows_for(89, 11, 1, 6144) != 1 || native_mmvq_il_rows_for(89, 12, 1, 6144) != 0) {
+ // sm_89 one column: only Q3_K with STRATA_MMVQ_IL_89=1 (the 4070 and the 4080 agree), else none; wider than 4: the plain kernels
+ if (native_mmvq_il_rows_for(89, 11, 1, 2560) != 0 || native_mmvq_il_rows_for(89, 11, 1, 6144) != (opt89 ? 1 : 0) ||
+ native_mmvq_il_rows_for(89, 12, 1, 6144) != 0) {
++bad;
std::printf("FAIL table: the sm_89 one-column row is not Q3_K only\n");
}