Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

NVIDIA sm_86 and sm_60: the measured bitwise decode switches on by default per architecture (nv_defaults; STRATA_SM86_PERF=0 / STRATA_SM60_PERF=0 turn them off)

Niko12212725c23f746e
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 758655a8a..ecc805986 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -561,7 +561,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
src/core/remote_expert_opt.cu
src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp
src/core/conversation_snapshot.cpp src/core/conversation_state.cpp src/core/conversation_memory.cpp src/core/peer_experts.cpp
- src/core/arch_defaults.cpp src/core/vmm.cpp src/core/conversation_file.cpp)
+ src/core/arch_defaults.cpp src/core/nv_defaults.cpp src/core/vmm.cpp src/core/conversation_file.cpp)
target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu ${_strata_gpu_runtime_target})
if(STRATA_BUILD_TESTS AND STRATA_ENABLE_CUDA AND NOT STRATA_HIP_GFX906)
diff --git a/docs/DETAILS.md b/docs/DETAILS.md
index 9951af43c..f38599c9c 100644
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@ -2217,6 +2217,27 @@ signal, early cleanup and `--errors` error-propagation checks. Its `--synchronou
mode is the negative control under memcheck: an eight-second watchdog releases
the original launch/service deadlock and the test fails.
+## RTX 30 series (sm_86) and Tesla P100 (sm_60): bitwise decode switches on by default
+
+The 0.1.45 switch set that is bit-exact on every card was measured switch by switch on a 12 GB RTX 3060, a 16 GB RTX A4000 (one card)
+and a 16 GB Tesla P100, IQ3_XXS, default flags (adaptive tier on), fresh engine per run, 8 interleaved pairs, equal expert-cache slots,
+with a noise arm: the whole 16-switch set gave +9.6% (3060), +14.6% (A4000) and +17.8% (P100) decode, 8/8 pairs each, and the identity
+gate (10 prompts + exact_pp 4K/20K, fixed flags) is IDENTICAL with it on. Removing one switch at a time (4 rounds each) kept the ones that
+paid on their architecture. They are on by default now, per architecture (`include/strata/core/nv_defaults.hpp`, set before the
+engine reads its switches; a switch you set yourself, to anything, is kept):
+
+- sm_86 (RTX 3050-3090, RTX A2000-A6000): `STRATA_ADAPT_NOWAIT=1`, `STRATA_ADAPT_LAG=2`, `STRATA_CPU_IQ_ALL_EXACT1=1`,
+ `STRATA_SIDE_MIXER=2`, `STRATA_ATTN_DAILY=1`, `STRATA_GR_UP75=1`, `STRATA_GU_SWIGLU=1`, `STRATA_QSA_ROWS=1`, `STRATA_B6_MMVQ_ROWS=1`.
+ `STRATA_SM86_PERF=0` turns the set off.
+- sm_60 (Tesla P100): `STRATA_ADAPT_NOWAIT=1`, `STRATA_ADAPT_LAG=2`, `STRATA_SIDE_MIXER=2`, `STRATA_FUSE_WAIT=2`, `STRATA_B6_MMVQ_ROWS=1`,
+ `STRATA_FUSE_DECODE=1`, `STRATA_GR_UP75=1`. `STRATA_SM60_PERF=0` turns the set off.
+
+The table is used when every visible card has that one compute capability. `STRATA_ADAPT_OVERLAP=1` (opt-in) does not run beside
+`STRATA_ADAPT_LAG` / `STRATA_ADAPT_NOWAIT`: set `STRATA_ADAPT_LAG=1 STRATA_ADAPT_NOWAIT=0` with it. Left out because they did not pay or
+were mixed: `STRATA_EXPDEC`, `STRATA_GR_NORM_FAST`, `STRATA_RAM_ADAPT`, `STRATA_KV_HOST_REGISTER`, `STRATA_HOST_CORE`,
+`STRATA_GR_DOWN_TG`, `STRATA_DEQUANT_WIDE`; `STRATA_EXCHANGE_ROTATE` never reaches READY on the 3060 and `STRATA_PCIE_BALANCE` is slower on a
+single card. Measured numbers: phaseA-tests/notes-148-nv.md.
+
## Compute capability 12.0 cards: faster decode (bitwise) and fused prompt experts by default
Measured on an RTX PRO 5000 Blackwell 72 GB (110 SMs, sm_120, Linux, CUDA 13.2), IQ3_S with every expert in VRAM, setup's
diff --git a/include/strata/core/nv_defaults.hpp b/include/strata/core/nv_defaults.hpp
new file mode 100644
index 000000000..d943f2875
--- /dev/null
+++ b/include/strata/core/nv_defaults.hpp
@@ -0,0 +1,30 @@
+// strata/core/nv_defaults.hpp - the per-architecture defaults of switches that are exact (bitwise) on NVIDIA sm_86 and sm_60.
+//
+// 148-nv measured every switch of the 0.1.45 bit-exact ON set, one at a time and in the stacked set, on an RTX 3060, an RTX A4000
+// (sm_86) and a Tesla P100 (sm_60): a switch is in a table only when its token ids equal the default's and it was faster on that
+// architecture (5+ interleaved pairs of fresh engines, noise arm, equal expert-cache slots; notes-148-nv.md). The identity gate
+// (10 prompts + exact_pp 4K/20K, fixed flags) is IDENTICAL with the whole table on. Switches that change bits, hang or lost speed
+// on a card (STRATA_EXCHANGE_ROTATE, STRATA_PCIE_BALANCE, STRATA_QSA_WARP, ...) are not in any table.
+//
+// STRATA_SM86_PERF=0 turns the sm_86 table off (cards of compute capability 8.6: RTX 3050-3090, RTX A2000-A6000)
+// STRATA_SM60_PERF=0 turns the sm_60 table off (Pascal GP100: Tesla P100)
+//
+// A switch the user set (to anything, including 0) is never overridden. Applied once at start, before any engine code reads its
+// switches; the table is used only when every visible CUDA device has the same compute capability.
+#pragma once
+
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace strata::core {
+
+/// The environment settings that are the defaults of a card with compute capability major.minor (raw, as the driver reports it);
+/// empty for every other card and when its off switch is 0.
+std::vector<std::pair<std::string, std::string>> nv_default_env(int major, int minor);
+
+/// Sets `nv_default_env(major, minor)` in this process's environment where the user has not set the variable, prints one line to
+/// stderr when it set anything, and returns the names it set.
+std::vector<std::string> apply_nv_defaults(int major, int minor);
+
+} // namespace strata::core
diff --git a/src/core/nv_defaults.cpp b/src/core/nv_defaults.cpp
new file mode 100644
index 000000000..4b1a9f983
--- /dev/null
+++ b/src/core/nv_defaults.cpp
@@ -0,0 +1,73 @@
+// src/core/nv_defaults.cpp - see include/strata/core/nv_defaults.hpp.
+#include "strata/core/nv_defaults.hpp"
+
+#include "strata/core/emulate.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+
+namespace strata::core {
+
+namespace {
+bool off(const char* name) {
+ const char* v = std::getenv(name);
+ return v != nullptr && v[0] == '0';
+}
+} // namespace
+
+std::vector<std::pair<std::string, std::string>> nv_default_env(int major, int minor) {
+ std::vector<std::pair<std::string, std::string>> t;
+ const int mj = strata::cc_major_of(major), mn = strata::cc_minor_of(minor);
+ if (mj == 8 && mn == 6) {
+ if (off("STRATA_SM86_PERF")) return t;
+ t = {
+ // 148-nv, RTX 3060 + RTX A4000 (IQ3_XXS, default flags, 8 pairs): the stack +9.6% / +14.6% (8/8 each), identity gate IDENTICAL.
+ // Each is in because removing it from the full stack cost speed on a card (drop-one, 4 rounds) and saved none on the other.
+ {"STRATA_ADAPT_NOWAIT", "1"}, // A4000 -6.3 points when removed (with ADAPT_LAG), 3060 -0.8
+ {"STRATA_ADAPT_LAG", "2"},
+ {"STRATA_CPU_IQ_ALL_EXACT1", "1"}, // A4000 -3.3, 3060 -0.6
+ {"STRATA_SIDE_MIXER", "2"}, // A4000 -1.2, 3060 -2.3
+ {"STRATA_ATTN_DAILY", "1"}, // -1.2, -1.3
+ {"STRATA_GR_UP75", "1"}, // -0.9, -1.8
+ {"STRATA_GU_SWIGLU", "1"}, // -0.8, -1.4
+ {"STRATA_QSA_ROWS", "1"}, // -0.5, -1.2
+ {"STRATA_B6_MMVQ_ROWS", "1"}, // A4000 0, 3060 -1.7
+ };
+ } else if (mj == 6 && mn == 0) {
+ if (off("STRATA_SM60_PERF")) return t;
+ t = {
+ // 148-nv, Tesla P100 (IQ3_XXS, default flags, 8 pairs): the stack +17.8% (8/8), identity gate IDENTICAL.
+ {"STRATA_ADAPT_NOWAIT", "1"}, // -10 points when removed (with ADAPT_LAG)
+ {"STRATA_ADAPT_LAG", "2"},
+ {"STRATA_SIDE_MIXER", "2"}, // -1.8
+ {"STRATA_FUSE_WAIT", "2"}, // -1.6
+ {"STRATA_B6_MMVQ_ROWS", "1"}, // -1.3
+ {"STRATA_FUSE_DECODE", "1"}, // -1.3
+ {"STRATA_GR_UP75", "1"}, // -1.2
+ };
+ }
+ return t;
+}
+
+std::vector<std::string> apply_nv_defaults(int major, int minor) {
+ std::vector<std::string> set;
+ for (const auto& kv : nv_default_env(major, minor)) {
+ if (std::getenv(kv.first.c_str()) != nullptr) continue; // the user's setting wins, whatever it is
+#if defined(_WIN32)
+ _putenv_s(kv.first.c_str(), kv.second.c_str());
+#else
+ setenv(kv.first.c_str(), kv.second.c_str(), 0);
+#endif
+ set.push_back(kv.first);
+ }
+ if (!set.empty()) {
+ const bool sm86 = strata::cc_major_of(major) == 8;
+ std::fprintf(stderr, "strata: %s: %zu exact speed switches on by default (%s=0 turns them off; a switch you set is kept): ",
+ sm86 ? "sm_86 (Ampere)" : "sm_60 (Pascal)", set.size(), sm86 ? "STRATA_SM86_PERF" : "STRATA_SM60_PERF");
+ for (size_t i = 0; i < set.size(); ++i) std::fprintf(stderr, "%s%s", i ? " " : "", set[i].c_str() + 7);
+ std::fprintf(stderr, "\n");
+ }
+ return set;
+}
+
+} // namespace strata::core
diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cu
index 1332335ac..d8cae2a6a 100644
--- a/src/kernels/cuda/iq_kernels.cu
+++ b/src/kernels/cuda/iq_kernels.cu
@@ -3479,7 +3479,9 @@ int env_int_or(const char* name, int dflt) {
const char* v = std::getenv(name);
return v != nullptr && v[0] != '\0' ? std::atoi(v) : dflt;
}
-int g_expdec = env_on("STRATA_EXPDEC") ? 1 : 0;
+// read at first use (not at load), so a per-architecture default set at start (nv_defaults.hpp) is seen; native_expert_set_expdec overrides
+int g_expdec = -1;
+int expdec_mode() { if (g_expdec < 0) g_expdec = env_on("STRATA_EXPDEC") ? 1 : 0; return g_expdec; }
int g_expdec_r = env_int_or("STRATA_EXPDEC_R", 1); // gate/up rows per lane: 1, 2 (bench: R=1 wins on the 2080 Ti)
int g_expdec_dr = env_int_or("STRATA_EXPDEC_DR", 4); // down rows per lane: 1, 2, 4 (Q2_0: R=4 wins on both cards)
int g_expdec_minb = env_int_or("STRATA_EXPDEC_MINB", STRATA_EXPDEC_MINB); // gate/up __launch_bounds__ min blocks: 2 (<= 128 regs) or 4 (<= 64)
@@ -3572,7 +3574,7 @@ void expdec_note(const NativeExpertLayout& L, bool gu, bool taken, cudaStream_t
bool expdec_gu(const NativeExpertLayout& L, int64_t gy, int64_t cap_groups, cudaStream_t s, const unsigned long long* grp_ptr,
const int32_t* grp_start, const int32_t* n_groups, const int32_t* ent_tok, const block_q8_1* X, float* gate,
float* up) {
- if (!g_expdec || g_old_kernels) return false;
+ if (!expdec_mode() || g_old_kernels) return false;
if (!expdec_gu_ok(L)) { expdec_note(L, true, false, s, gy, cap_groups); return false; }
expdec_note(L, true, true, s, gy, cap_groups);
switch (L.gu_type) {
@@ -3604,7 +3606,7 @@ void expdec_down_t(int64_t gy, int64_t cap_groups, cudaStream_t s, const unsigne
}
bool expdec_down(const NativeExpertLayout& L, int64_t gy, int64_t cap_groups, cudaStream_t s, const unsigned long long* grp_ptr,
const int32_t* grp_start, const int32_t* n_groups, const int32_t* ent_dst, const block_q8_1* hq, float* out) {
- if (!g_expdec || g_old_kernels) return false;
+ if (!expdec_mode() || g_old_kernels) return false;
if (!expdec_down_ok(L)) { expdec_note(L, false, false, s, gy, cap_groups); return false; }
expdec_note(L, false, true, s, gy, cap_groups);
if (L.d_type == 20) expdec_down_t<20>(gy, cap_groups, s, grp_ptr, grp_start, n_groups, ent_dst, hq, L, out);
@@ -4474,7 +4476,7 @@ bool g_grouped_v1 = env_on("STRATA_GROUPED_V1");
void native_grouped_set_v1(bool v1) { g_grouped_v1 = v1; }
void native_expert_set_expdec(int on) { g_expdec = on ? 1 : 0; }
-int native_expert_expdec() { return g_expdec; }
+int native_expert_expdec() { return expdec_mode(); }
void native_expert_set_expdec_params(const NativeExpertExpdecParams& p) {
if (p.gu_rows_per_lane > 0) g_expdec_r = p.gu_rows_per_lane;
if (p.down_rows_per_lane > 0) g_expdec_dr = p.down_rows_per_lane;
@@ -4608,7 +4610,7 @@ void native_expert_grouped(const NativeExpertLayout& L, const unsigned long long
#else
const bool gs_layout = true;
#endif
- if (gs_layout && g_exp_phase == 0 && !v1 && gu_swiglu_on() && !(g_expdec && !g_old_kernels && expdec_gu_ok(L))) {
+ if (gs_layout && g_exp_phase == 0 && !v1 && gu_swiglu_on() && !(expdec_mode() && !g_old_kernels && expdec_gu_ok(L))) {
switch (L.gu_type) {
#define STRATA_GS(T) case T: gs_done = launch_gu_swiglu<T>(L, gy, s, grp_ptr, grp_start, n_groups, ent_tok, X, hq); break;
STRATA_GU_FMTS(STRATA_GS)
diff --git a/src/kernels/cuda/native_mmvq.cu b/src/kernels/cuda/native_mmvq.cu
index 57e1c377b..05f9b42d4 100644
--- a/src/kernels/cuda/native_mmvq.cu
+++ b/src/kernels/cuda/native_mmvq.cu
@@ -1156,7 +1156,9 @@ struct SmallTraits<IQ4NLBlock, 4> {
// equal to ncols = 1 only to float rounding (the cross-warp reduction groups partial sums differently).
bool g_multi_exact = true; // until the upstream layout is timed on an idle GPU (plan rule: default only what is measured)
// B6 STRATA_B6_MMVQ_ROWS (lab, default 0 = off): see native_mmvq_set_b6_rows
-int g_b6_rows = [] { const char* v = std::getenv("STRATA_B6_MMVQ_ROWS"); return v ? std::atoi(v) : 0; }();
+// read at first use (not at load): a per-architecture default set at start (nv_defaults.hpp) must be seen
+int g_b6_rows = -1;
+int b6_rows_mode() { if (g_b6_rows < 0) { const char* v = std::getenv("STRATA_B6_MMVQ_ROWS"); g_b6_rows = v ? std::atoi(v) : 0; } return g_b6_rows; }
// S26 STRATA_TSUM=1 (TS): warp 0's NCOLS x ROWS sums as one transposed butterfly (s26_tsum.cuh), bitwise the same
static bool s26_tsum_on() {
@@ -1456,8 +1458,8 @@ bool b6_launch(const typename F::Block* w, const Q81Block* x, float* y, int n_in
if constexpr (!B6X<F>::ok) {
return false;
} else {
- if (g_b6_rows <= 0 || !g_multi_exact || s26_tsum_on() || n_in / F::DIV < 1) return false;
- const int rows = g_b6_rows == 1 ? 2 : g_b6_rows;
+ if (b6_rows_mode() <= 0 || !g_multi_exact || s26_tsum_on() || n_in / F::DIV < 1) return false;
+ const int rows = b6_rows_mode() == 1 ? 2 : b6_rows_mode();
const dim3 threads(WARP, WARPS);
if (rows == 2)
b6_mmvq_rows_kernel<F, NCOLS, 2><<<unsigned((n_out + 1) / 2), threads, 0, s>>>(w, x, y, n_in, n_out);
@@ -2441,7 +2443,7 @@ void native_q6_k_unpack(const void* weights) {
void native_mmvq_set_multi_exact(bool exact) { g_multi_exact = exact; }
bool native_mmvq_multi_exact() { return g_multi_exact; }
void native_mmvq_set_b6_rows(int mode) { g_b6_rows = mode; }
-int native_mmvq_b6_rows() { return g_b6_rows; }
+int native_mmvq_b6_rows() { return b6_rows_mode(); }
std::size_t native_q8_1_bytes(int n_in, int ncols) {
validate_shape(n_in, ncols);
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index aae516316..1f944c16e 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -17,6 +17,7 @@
#include "strata/platform/memory.hpp"
#include "strata/core/arch_defaults.hpp"
+#include "strata/core/nv_defaults.hpp"
#include "strata/kernels/gfx_arch.hpp"
#include "strata/core/dma_batch.hpp"
#include "strata/core/device.hpp"
@@ -2646,6 +2647,24 @@ int main(int argc, char** argv) {
else (void) cudaGetLastError();
}
#endif
+#if !defined(STRATA_USE_HIP) && !defined(STRATA_GPU_METAL)
+ { // CUDA sm_86 / sm_60: the switches that are exact AND faster there are on by default (strata/core/nv_defaults.hpp), when every
+ // visible card has that one compute capability; before any engine code reads its switches, the user's own settings are kept
+ int nd = 0;
+ if (cudaGetDeviceCount(&nd) == cudaSuccess && nd > 0) {
+ int m0 = 0, n0 = 0;
+ bool same = cudaDeviceGetAttribute(&m0, cudaDevAttrComputeCapabilityMajor, 0) == cudaSuccess &&
+ cudaDeviceGetAttribute(&n0, cudaDevAttrComputeCapabilityMinor, 0) == cudaSuccess;
+ for (int d = 1; same && d < nd; ++d) {
+ int m = 0, n = 0;
+ same = cudaDeviceGetAttribute(&m, cudaDevAttrComputeCapabilityMajor, d) == cudaSuccess &&
+ cudaDeviceGetAttribute(&n, cudaDevAttrComputeCapabilityMinor, d) == cudaSuccess && m == m0 && n == n0;
+ }
+ if (same) strata::core::apply_nv_defaults(m0, n0);
+ }
+ (void) cudaGetLastError();
+ }
+#endif
#if defined(STRATA_USE_HIP) && !defined(_WIN32)
if (o.prefill_auto_bare && o.prefill_auto_cap == 0) {
// gfx1200 / gfx1201 (RDNA4, Linux): a bare `--prefill auto` may reach 16384 instead of 8192 (R9700, IQ3_S / Q2_0, 16K / 32K /