Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
sm_120: four bitwise decode switches on by default on a 12.0 card
Niko1221ba2c2d5fb766
diff --git a/include/strata/core/sm120_series.hpp b/include/strata/core/sm120_series.hppnew file mode 100644index 000000000..6777cdbf3--- /dev/null+++ b/include/strata/core/sm120_series.hpp@@ -0,0 +1,38 @@+// strata/core/sm120_series.hpp - bitwise kernel switches that are ON by default on a compute capability 12.0 card+// (RTX 50 series / RTX PRO Blackwell workstation cards).+//+// 147-verify measured these exact switches on an RTX PRO 5000 Blackwell 72 GB (IQ3_S, every expert in VRAM): together+// a verify round of 4 tokens 13.70 -> 12.23 ms with the same text (notes-147-l620.md). Each one is bitwise equal to+// the kernels it replaces (their own parity tests and the identity gate), so only the speed changes:+// STRATA_SIDE_MIXER=2 the mixer's input-only work on branches of the verify graph+// STRATA_ATTN_DAILY=1 the Daily's decode attention (attn_chunk_kernel_pre75 + merge v2)+// STRATA_GR_UP75=1 the hyper-connection up projection with its rows' loads hoisted+// STRATA_GR_DOWN_TG=2 the hyper-connection down projection as two token groups (twice the blocks)+//+// STRATA_SM120_PERF=0 turns the whole set off on a 12.0 card (the defaults every other card has).+// Each switch's own variable still wins in both directions (=0 off, =1 / =2 on), on any card, as before.+// Nothing here changes a card that is not a 12.0 (an sm_121 / sm_100 card keeps its defaults).+#pragma once++#include "strata/core/emulate.hpp"++#include <cstdlib>++namespace strata {++/// STRATA_SM120_PERF (read once): unset or anything but "0" = on+inline bool sm120_series() {+ static const bool on = [] {+ const char* e = std::getenv("STRATA_SM120_PERF");+ return e == nullptr || std::atoi(e) != 0;+ }();+ return on;+}++/// the device (raw cudaDevAttrComputeCapabilityMajor / Minor) is a 12.0 card, or emulates one (STRATA_EMULATE_CC=120)+inline bool is_sm120(int major, int minor) { return cc_major_of(major) == 12 && cc_minor_of(minor) == 0; }++/// the default of a bitwise switch of the set: on for a 12.0 card unless STRATA_SM120_PERF=0+inline bool sm120_default(int major, int minor) { return is_sm120(major, minor) && sm120_series(); }++} // namespace stratadiff --git a/src/core/verify.cpp b/src/core/verify.cppindex 52a96be44..23580e7d7 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -1,6 +1,7 @@// src/core/verify.cpp - see include/strata/core/verify.hpp.#include "strata/core/batch_rows.hpp"#include "strata/core/sm75_series.hpp"+#include "strata/core/sm120_series.hpp"#include "strata/kernels/gfx_arch.hpp"#include "strata/core/verify.hpp"#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)@@ -176,9 +177,28 @@ inline bool g_qfuse_hc() {return false;#endif}-// fork 34a35db + b063c13 (Eddoursul), STRATA_SIDE_MIXER=1 / =2 (opt-in): the mixer's work that reads only the layer's-// input on branches of the window's graph (bitwise the same outputs); 0 / unset = in line, and no branch stream is created-inline int g_side_mixer() { static const int m = [] { const char* v = std::getenv("STRATA_SIDE_MIXER"); return v ? std::max(0, std::min(2, std::atoi(v))) : 0; }(); return m; }+// fork 34a35db + b063c13 (Eddoursul), STRATA_SIDE_MIXER=1 / =2: the mixer's work that reads only the layer's input on+// branches of the window's graph (bitwise the same outputs); 0 = in line, and no branch stream is created. Unset: 2 on a+// compute capability 12.0 card (sm120_series.hpp; STRATA_SM120_PERF=0 turns it off), 0 elsewhere.+inline int g_side_mixer() {+ static const int env = [] { const char* v = std::getenv("STRATA_SIDE_MIXER"); return v ? std::max(0, std::min(2, std::atoi(v))) : -1; }();+ if (env >= 0) return env;+#if defined(STRATA_USE_HIP) || defined(STRATA_GPU_METAL)+ return 0;+#else+ static int per_dev[64]; // 0 unknown, else mode + 1+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return 0; }+ if (!per_dev[dev]) {+ int maj = 0, mnr = 0;+ const bool ok = cudaDeviceGetAttribute(&maj, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&+ cudaDeviceGetAttribute(&mnr, cudaDevAttrComputeCapabilityMinor, dev) == cudaSuccess;+ if (!ok) cudaGetLastError();+ per_dev[dev] = 1 + (ok && strata::sm120_default(maj, mnr) ? 2 : 0);+ }+ return per_dev[dev] - 1;+#endif+}inline bool g_qfuse() { static const bool on = [] { const char* v = std::getenv("STRATA_QFUSE"); return v != nullptr && std::atoi(v) != 0; }(); return on; }#define VDBG(...) do { if (g_dbg) { std::fprintf(stderr, "verify dbg: " __VA_ARGS__); std::fflush(stderr); } } while (0)diff --git a/src/kernels/cuda/fused_gr.cu b/src/kernels/cuda/fused_gr.cuindex be1bfe8b4..690ea955c 100644--- a/src/kernels/cuda/fused_gr.cu+++ b/src/kernels/cuda/fused_gr.cu@@ -1,6 +1,7 @@// src/kernels/cuda/fused_gr.cu - see include/strata/kernels/fused_gr.hpp.#include "strata/core/emulate.hpp"#include "strata/core/sm75_series.hpp"+#include "strata/core/sm120_series.hpp"#include "strata/kernels/q8_1_finite.hpp" // #606: q8_1_ds#include "strata/core/geometry.hpp"#include "strata/kernels/fused_gr.hpp"@@ -1202,7 +1203,8 @@ bool up75_on() {int major = 0, minor = 0;cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);- per_dev[dev] = strata::sm75_default(major, minor) ? 2 : 1; // Turing, STRATA_SM75_PERF=1+ // Turing (STRATA_SM75_PERF=1), and a 12.0 card (sm120_series.hpp: RTX PRO 5000, a 4-token verify round -1.7%)+ per_dev[dev] = strata::sm75_default(major, minor) || strata::sm120_default(major, minor) ? 2 : 1;}return per_dev[dev] == 2;#endif@@ -1219,15 +1221,17 @@ int down_tg(int ct) {if (env >= 0) return env > 0 && env <= 3 && ct % env == 0 ? env : 0;// fused_gr_bench cold on an RTX 2080 Ti (down incl. a stamp): T 4 29.1 -> 27.3 us, T 6 38.5 -> 32.2 with pairs;// one token per block is slower at every T (the weights are read from L2 once per group)- static int per_dev[64]; // 0 unknown, 1 no, 2 sm_75+ static int per_dev[64]; // 0 unknown, 1 no, 2 sm_75, 3 a 12.0 cardint dev = 0;if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) return 0;if (!per_dev[dev]) {int major = 0, minor = 0;cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, dev);cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, dev);- per_dev[dev] = strata::sm75_default(major, minor) ? 2 : 1; // Turing, STRATA_SM75_PERF=1+ per_dev[dev] = strata::sm75_default(major, minor) ? 2 : strata::sm120_default(major, minor) ? 3 : 1; // Turing, STRATA_SM75_PERF=1}+ // a 12.0 card (sm120_series.hpp): token pairs at every even launch (RTX PRO 5000, a 4-token round -2.4%)+ if (per_dev[dev] == 3) return ct % 2 == 0 ? 2 : 0;return per_dev[dev] == 2 && (ct == 4 || ct == 6) ? 2 : 0;#endif}diff --git a/src/kernels/cuda/qsa_decode_attn.cu b/src/kernels/cuda/qsa_decode_attn.cuindex 854b479ee..fbbef636e 100644--- a/src/kernels/cuda/qsa_decode_attn.cu+++ b/src/kernels/cuda/qsa_decode_attn.cu@@ -1,6 +1,7 @@// src/kernels/cuda/qsa_decode_attn.cu - see include/strata/kernels/qsa_decode_attn.hpp.#include "strata/kernels/qsa_decode_attn.hpp"#include "strata/core/sm75_series.hpp"+#include "strata/core/sm120_series.hpp"#include "strata/core/geometry.hpp"#include "strata/kernels/kv_q8.hpp"#include "strata/kernels/kv_q4.hpp"@@ -832,15 +833,29 @@ bool query_swizzle_gfx906_device() {// pass on PR #540's kernel (attn_chunk_kernel_pre75<M, 2, false>, the Daily's attn_chunk_kernel), bit for bit the// default kernel's partials. CUDA only; unset or anything but "1" keeps upstream 0.1.42's kernels.// STRATA_ATTN_PRE75=0 still turns the chunk kernel off (A/B).+// 147-verify: unset, it is on for a compute capability 12.0 card (sm120_series.hpp; RTX PRO 5000, a 4-token verify round+// -2.6%, bitwise); STRATA_ATTN_DAILY=0 (anything but "1") keeps upstream's kernels there too.[[maybe_unused]] bool daily_attn() {#if defined(__HIPCC__)return false;#else- static const bool on = [] {+ static const int env = [] {const char* v = std::getenv("STRATA_ATTN_DAILY");- return v != nullptr && v[0] == '1' && v[1] == '\0';+ if (v == nullptr) return -1;+ return v[0] == '1' && v[1] == '\0' ? 1 : 0;}();- return on;+ if (env >= 0) return env != 0;+ static int per_dev[64]; // 0 unknown, 1 no, 2 yes+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess || dev < 0 || dev >= 64) { cudaGetLastError(); return false; }+ if (!per_dev[dev]) {+ int maj = 0, mnr = 0;+ const bool ok = cudaDeviceGetAttribute(&maj, cudaDevAttrComputeCapabilityMajor, dev) == cudaSuccess &&+ cudaDeviceGetAttribute(&mnr, cudaDevAttrComputeCapabilityMinor, dev) == cudaSuccess;+ if (!ok) cudaGetLastError();+ per_dev[dev] = ok && strata::sm120_default(maj, mnr) ? 2 : 1;+ }+ return per_dev[dev] == 2;#endif}@@ -910,13 +925,11 @@ int attn_kv_mode(const QsaAttnPools& p) {// STRATA_ATTN_MERGE_V2: 1 merges with attn_merge_v2_kernel, 0 with attn_merge_kernel. Unset it follows// STRATA_ATTN_DAILY (the Daily had it on by default on NVIDIA), so upstream's merger stays the default.int attn_merge_v2() {- static const int on = [] {- int r = daily_attn() ? 1 : 0;+ static const int env = [] {const char* v = std::getenv("STRATA_ATTN_MERGE_V2");- if (v != nullptr && *v != '\0') r = std::atoi(v) != 0 ? 1 : 0;- return r;+ return v != nullptr && *v != '\0' ? (std::atoi(v) != 0 ? 1 : 0) : -1;}();- return on;+ return env >= 0 ? env : (daily_attn() ? 1 : 0);}// the merge of `n_q` queries' partials: v2 = 0 attn_merge_kernel, 1 attn_merge_v2_kernel (bitwise the same output)