Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

CUDA Plan opt-ins become defaults where measured: STRATA_GR_V3 on a discrete 12.0 card above 13 GiB, STRATA_PROMPT_ATTN_IMMA on CUDA sm_80+ (env =0 turns each off); DeepSeek Faster mode (5 experts) by default on 2+ RDNA4 cards that hold half the experts; the route-prior/GR_V3/IMMA rules are pure functions with tests; gr_parity and fused_gr_bench pin STRATA_GR_V3=0 for their bitwise contract

Niko122154d9363c972c
diff --git a/include/strata/core/sm120_series.hpp b/include/strata/core/sm120_series.hpp
index 9059d312e..b1cc11925 100644
--- a/include/strata/core/sm120_series.hpp
+++ b/include/strata/core/sm120_series.hpp
@@ -42,4 +42,27 @@ inline bool is_sm120(int major, int minor) { return cc_major_of(major) == 12 &&
/// the default of a bitwise switch of the set: on for a 12.0 card unless STRATA_SM120_PERF=0
inline bool sm120_default(int major, int minor) { return is_sm120(major, minor) && sm120_series(); }
+// ---- 0.2.7 defaults of the CUDA speed work (pure rules; the engine reads the device and calls these; tests in arch_defaults_test)
+// `hip`: a HIP build, where a gfx1200 / gfx1201 reports compute capability 12.0 as well and none of these is measured or wanted.
+// `total_bytes`: the card's memory (cudaDeviceProp::totalGlobalMem); `integrated`: a GPU that shares the PC's memory.
+constexpr double kSmallCardGiB = 13.0; // "a 12 GB card": 12 GB cards report 11.6 - 12.0 GiB, 16 GB ones 15.6 - 16.0
+
+/// STRATA_ROUTE_PRIOR when the user set nothing: 2.0 (residency-biased routing, changes which experts run; owner-approved for 12 GB cards) on a
+/// discrete CUDA card of 13 GiB or less, 0 everywhere else. Measured on an RTX 3060 12 GB only.
+inline float route_prior_default(bool hip, bool integrated, double total_bytes) {
+ return !hip && !integrated && total_bytes > 0.0 && total_bytes / 1073741824.0 <= kSmallCardGiB ? 2.0f : 0.0f;
+}
+
+/// STRATA_GR_V3 (the stream-split hyper-connection read, another summation order) when the user set nothing: ON for a discrete compute capability
+/// 12.0 card (STRATA_SM120_PERF=0 turns the set off) with more than 13 GiB, CUDA builds only. Measured: RTX PRO 5000 72 GB +1.2% decode clean,
+/// -3.0% / -5.9% on 2x RTX A4000 (sm_86), +1.0% / -3.4% on an RTX 3060 (sm_86): so not those; a 12 GB sm_120 card is unmeasured.
+inline bool gr_v3_default(bool hip, int major, int minor, bool integrated, double total_bytes) {
+ return !hip && !integrated && sm120_default(major, minor) && total_bytes / 1073741824.0 > kSmallCardGiB;
+}
+
+/// STRATA_PROMPT_ATTN_IMMA (the prompt attention on int8 tensor cores, another summation order) when the user set nothing: ON for every CUDA card of
+/// compute capability 8.0 or newer (the int8 m16n8k32 MMA), not Turing / Volta / Pascal, not HIP. `cc` = 10 * major + minor as the engine
+/// resolves it (an emulated or forced Turing counts as 75). Measured on sm_120 (RTX PRO 5000) and sm_86 (2x RTX A4000): +1.1 .. +1.8% prompt.
+inline bool prompt_attn_imma_default(bool hip, int cc) { return !hip && cc >= 80; }
+
} // namespace strata
diff --git a/src/core/arch_defaults_test.cpp b/src/core/arch_defaults_test.cpp
index 459fdaa49..9723b2b78 100644
--- a/src/core/arch_defaults_test.cpp
+++ b/src/core/arch_defaults_test.cpp
@@ -1,6 +1,7 @@
// src/core/arch_defaults_test.cpp - the per-architecture default table (no GPU needed).
#include "strata/core/arch_defaults.hpp"
#include "strata/core/gfx12_platform_defaults.hpp"
+#include "strata/core/sm120_series.hpp"
#include <cstdio>
#include <cstdlib>
@@ -226,5 +227,46 @@ int main() {
unset("STRATA_UMA_REGISTER");
// both platforms: no Windows guard on the iGPU set (the owner wants it there too; untested on Windows is only said in the log)
std::printf(fails ? "arch_defaults_test: %d FAILED\n" : "arch_defaults_test: OK\n", fails);
+ // ---- 0.2.7 CUDA defaults (sm120_series.hpp): which cards get which, and which never do
+ {
+ const double GiB = 1073741824.0;
+ using namespace strata;
+ // residency-biased routing: 2.0 on a discrete CUDA card of 13 GiB or less, nothing else
+ CHECK(route_prior_default(false, false, 11.9 * GiB) == 2.0f); // RTX 3060 / 5070 12 GB
+ CHECK(route_prior_default(false, false, 12.0 * GiB) == 2.0f);
+ CHECK(route_prior_default(false, false, 8.0 * GiB) == 2.0f); // an 8 GB card
+ CHECK(route_prior_default(false, false, 13.0 * GiB) == 2.0f); // the boundary is inclusive
+ CHECK(route_prior_default(false, false, 13.5 * GiB) == 0.0f);
+ CHECK(route_prior_default(false, false, 15.7 * GiB) == 0.0f); // 16 GB card: z440's A4000, P100
+ CHECK(route_prior_default(false, false, 72.0 * GiB) == 0.0f); // RTX PRO 5000
+ CHECK(route_prior_default(true, false, 12.0 * GiB) == 0.0f); // a HIP build: a 12 GB RX 6700 XT / 7700 XT
+ CHECK(route_prior_default(true, true, 12.0 * GiB) == 0.0f); // Strix Halo with a small carve-out
+ CHECK(route_prior_default(false, true, 8.0 * GiB) == 0.0f); // an integrated GPU on CUDA (Jetson-like)
+ CHECK(route_prior_default(false, false, 0.0) == 0.0f); // unknown memory
+ // the stream-split hyper-connection read: a discrete 12.0 card above 13 GiB (RTX PRO 5000 72 GB, RTX 5090 32 GB), nothing else
+ CHECK(gr_v3_default(false, 12, 0, false, 72.0 * GiB));
+ CHECK(gr_v3_default(false, 12, 0, false, 31.4 * GiB)); // RTX 5090
+ CHECK(!gr_v3_default(false, 12, 0, false, 11.9 * GiB)); // RTX 5070 12 GB: streams most experts, unmeasured
+ CHECK(gr_v3_default(false, 12, 0, false, 15.6 * GiB)); // a 16 GB sm_120 card (RTX 5060 Ti / 5070 Ti): above 13 GiB, on (unmeasured size, same arch)
+ CHECK(!gr_v3_default(false, 8, 6, false, 15.7 * GiB)); // 2x RTX A4000 (sm_86): measured -3.0% / -5.9%
+ CHECK(!gr_v3_default(false, 8, 6, false, 72.0 * GiB)); // any sm_86
+ CHECK(!gr_v3_default(false, 8, 9, false, 24.0 * GiB)); // sm_89
+ CHECK(!gr_v3_default(false, 12, 1, false, 128.0 * GiB)); // sm_121 (DGX Spark class): not 12.0
+ CHECK(!gr_v3_default(false, 12, 0, true, 128.0 * GiB)); // integrated
+ CHECK(!gr_v3_default(true, 12, 0, false, 32.0 * GiB)); // a HIP build: gfx1201 reports 12.0 too
+ CHECK(!gr_v3_default(false, 6, 0, false, 16.0 * GiB)); // P100
+ }
+ {
+ using namespace strata;
+ // the prompt attention on int8 tensor cores: every CUDA card from sm_80 on, not Turing / Volta / Pascal, not HIP
+ CHECK(prompt_attn_imma_default(false, 120)); // RTX PRO 5000 / RTX 50: measured +1.6-1.8%
+ CHECK(prompt_attn_imma_default(false, 86)); // 2x RTX A4000 / RTX 3060: measured +1.1-1.3%
+ CHECK(prompt_attn_imma_default(false, 80));
+ CHECK(prompt_attn_imma_default(false, 89));
+ CHECK(!prompt_attn_imma_default(false, 75)); // Turing (also a forced STRATA_QSA_WARP=1 card)
+ CHECK(!prompt_attn_imma_default(false, 70)); // Volta
+ CHECK(!prompt_attn_imma_default(false, 60)); // Pascal (P100)
+ CHECK(!prompt_attn_imma_default(true, 120)); // HIP
+ }
return fails ? 1 : 0;
}
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index b9d4eeded..f99dc9e8b 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -225,7 +225,7 @@ inline float g_route_prior() {
cudaDeviceProp p;
const bool ok = cudaGetDeviceProperties(&p, dev) == cudaSuccess;
if (!ok) cudaGetLastError();
- per_dev[dev] = 1.0f + (ok && !p.integrated && (double) p.totalGlobalMem / 1073741824.0 <= 13.0 ? 2.0f : 0.0f);
+ per_dev[dev] = 1.0f + (ok ? strata::route_prior_default(false, p.integrated != 0, (double) p.totalGlobalMem) : 0.0f);
}
return per_dev[dev] - 1.0f;
#endif
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hpp
index 140b3df4a..ae872def0 100644
--- a/src/dsv41/ds4_experts_rule.hpp
+++ b/src/dsv41/ds4_experts_rule.hpp
@@ -18,10 +18,13 @@ constexpr int kFasterModeExperts = 5;
// requested: --experts (-1: not given, the rule decides); vram_bytes: the total memory of the cards the engine uses;
// model_k: the experts a word takes in the model; integrated: a GPU that shares the PC's RAM (Strix Halo: part of the experts live only on
// the SSD, each expert dropped is less read - measured 5 against 6 on a Radeon 8065S: decode +11..13%, prompts +5..7%). The result is in 1 .. model_k.
-inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k, bool integrated = false) {
+// multi_rdna4: two or more RDNA4 cards (gfx1200 / gfx1201: R9700, RX 9000) that together hold at least half of the routed experts - the 4 x R9700 box
+// the Marlin stream measured: 5 against 6 experts, 2.25 bpw, 8K prompt 1339 -> 1419 ppts (+6%) and decode 78.1 -> 81.2 tok/s (+4%), reference KL
+// 0.10987 against 0.10833 and top-1 87.59% against 87.53% (the gate is 0.006). Faster mode there too, by default.
+inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k, bool integrated = false, bool multi_rdna4 = false) {
if (model_k <= 1) return model_k;
int n = requested;
- if (n < 1) n = integrated || (vram_bytes != 0 && vram_bytes <= kFasterModeVramMax) ? kFasterModeExperts : model_k;
+ if (n < 1) n = integrated || multi_rdna4 || (vram_bytes != 0 && vram_bytes <= kFasterModeVramMax) ? kFasterModeExperts : model_k;
if (n > model_k) n = model_k;
return n < 1 ? 1 : n;
}
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index e36ed7673..9c3596441 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -2132,11 +2132,13 @@ struct Model::Impl {
cudaDeviceProp ip{};
if (cudaGetDeviceProperties(&ip, dv.id) == cudaSuccess && ip.integrated) integrated = true;
}
- const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp, integrated), l.k_exp);
+ // two or more RDNA4 cards holding at least half of the experts (the 4 x R9700 class): Faster by default too
+ const bool multi_rdna4 = rdna4() && devs.size() >= 2 && gpu_expert_share() >= 0.5;
+ const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp, integrated, multi_rdna4), l.k_exp);
experts_used = n;
if (n < l.k_exp)
std::fprintf(stderr, "ds4: experts per word: %d (%s; full quality: --experts %d)\n", n,
- o.experts >= 1 ? "as asked" : "faster mode", l.k_exp);
+ o.experts >= 1 ? "as asked" : multi_rdna4 ? "faster mode: the cards hold most of the experts" : "faster mode", l.k_exp);
else if (o.experts >= 1)
std::fprintf(stderr, "ds4: experts per word: %d (full quality)\n", n);
}
diff --git a/src/kernels/cuda/fused_gr.cu b/src/kernels/cuda/fused_gr.cu
index 73a9089b7..ad1fcbd34 100644
--- a/src/kernels/cuda/fused_gr.cu
+++ b/src/kernels/cuda/fused_gr.cu
@@ -2302,6 +2302,32 @@ int g_gr_fast = -1;
#endif
void fused_gr_set_fast(int on) { g_gr_fast = on; }
+// STRATA_GR_V3: the stream-split hyper-connection read (gr_down_v3_kernel / gr_up_v3_kernel). Another summation order than the default
+// read (~1e-7; a greedy tie can flip, so the answers are not byte-identical to 0.2.6's). 0.2.7: ON by default on a compute capability 12.0
+// card (RTX 50 / RTX PRO Blackwell: sm120_series.hpp, STRATA_SM120_PERF=0 turns the set off) with more than 13 GiB of memory, discrete;
+// measured on the RTX PRO 5000 72 GB with every expert resident: code decode 245.1 -> 248.1 tok/s (+1.2%, identical tokens), a story prompt
+// +6.4% (tokens changed, not a claim), held-out document prompt +4.3%, with STRATA_PROMPT_ATTN_IMMA=1 +1.5% decode / +1.6-1.8% prompt together,
+// 5 pairs. NOT a default elsewhere: it measured -3.0% / -5.9% on 2x RTX A4000 (sm_86) and +1.0% / -3.4% on an RTX 3060 (sm_86), and a 12 GB
+// sm_120 card (RTX 5070: streams most experts, the read is a small share of its window) is unmeasured. STRATA_GR_V3=0 / 1 forces it off / on
+// on any card; never in a HIP build (gfx12 reports compute capability 12.0 too). Decided per card, first asked in fused_gr_check (before any
+// graph capture).
+static bool gr_v3_default_on(int dev) {
+#if defined(__HIPCC__)
+ (void) dev;
+ return false;
+#else
+ static int per_dev[64]; // 0 unknown, 1 no, 2 yes
+ if (dev < 0 || dev >= 64) return false;
+ if (!per_dev[dev]) {
+ cudaDeviceProp p;
+ const bool ok = cudaGetDeviceProperties(&p, dev) == cudaSuccess;
+ if (!ok) cudaGetLastError();
+ per_dev[dev] = ok && strata::gr_v3_default(false, p.major, p.minor, p.integrated != 0, (double) p.totalGlobalMem) ? 2 : 1;
+ }
+ return per_dev[dev] == 2;
+#endif
+}
+
bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream, unsigned long long* stamp_buf,
int stamp_i0) {
if (n_tok < 1 || n_tok > kFusedGrMaxT || xn_scratch == nullptr) {
@@ -2337,12 +2363,13 @@ bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi
}
return q8;
}
- // STRATA_GR_V3=1: the two-kernel read above (another summation order - opt-in)
- static const bool v3 = [] { const char* v = std::getenv("STRATA_GR_V3"); return v != nullptr && std::atoi(v) != 0; }();
+ // the two-kernel read above (another summation order): STRATA_GR_V3=0 / 1, unset = gr_v3_default_on (a 12.0 card with > 13 GiB)
+ static const int v3_env = [] { const char* v = std::getenv("STRATA_GR_V3"); return v != nullptr ? (std::atoi(v) != 0 ? 1 : 0) : -1; }();
static int split3[64] = {}; // per device: 0 = not decided yet, 1 / 2 = column halves S, -1 = does not fit
int dev3 = 0;
+ cudaGetDevice(&dev3);
+ const bool v3 = v3_env >= 0 ? v3_env == 1 : gr_v3_default_on(dev3);
if (v3) {
- cudaGetDevice(&dev3);
if (dev3 >= 0 && dev3 < 64 && split3[dev3] == 0) {
int optin = 0;
cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev3);
@@ -2653,6 +2680,7 @@ int fused_gr_variant() {
void fused_gr_check() {
int dev = 0;
cudaGetDevice(&dev);
+ (void) gr_v3_default_on(dev); // decided here, before any graph capture asks
if (dev < 0 || dev >= 64 || g_variant[dev].load() > 0) return;
const int want = env_variant();
if (want == kHcPlain) {
diff --git a/src/kernels/cuda/qsa_prompt_attn.cu b/src/kernels/cuda/qsa_prompt_attn.cu
index 3bdc4ffc9..f2b2d0a06 100644
--- a/src/kernels/cuda/qsa_prompt_attn.cu
+++ b/src/kernels/cuda/qsa_prompt_attn.cu
@@ -1,5 +1,6 @@
// src/kernels/cuda/qsa_prompt_attn.cu - see include/strata/kernels/qsa_prompt_attn.hpp.
#include "strata/core/emulate.hpp"
+#include "strata/core/sm120_series.hpp"
#include "strata/core/sm75_series.hpp"
#include "strata/core/geometry.hpp"
#include "strata/core/gfx12_platform_defaults.hpp"
@@ -2418,12 +2419,16 @@ bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int3
if (pf_env == 1 || (pf_env < 0 && t75 && strata::sm75_series())) return launch<1, true>(q, pools, ids, steps, cap, s, attn, n_q, st);
return launch<1>(q, pools, ids, steps, cap, s, attn, n_q, st);
}
- // STRATA_PROMPT_ATTN_IMMA=1 (opt-in, sm_80+): both products on INT8 tensor cores (prompt_attn_i8v3_kernel)
+ // STRATA_PROMPT_ATTN_IMMA (sm_80+): both products on INT8 tensor cores (prompt_attn_i8v3_kernel). ON by default since 0.2.7 on every
+ // card that has them (Ampere and newer, CUDA builds; not Turing, Volta or Pascal, which have no int8 m16n8k32 MMA this kernel needs):
+ // +1.6% / +1.8% prompt at 8K / 16K on the RTX PRO 5000 (sm_120) and +1.1% / +1.3% at 8K / 16K on 2x RTX A4000 (sm_86), decode unchanged,
+ // 5 pairs each; another summation order (~1e-7), so a near-tie can flip a prompt's first token. Sm_80, 8.9, 9.0 and the datacentre
+ // Blackwells run the same kernel unmeasured. STRATA_PROMPT_ATTN_IMMA=0 is the off switch (the v2 i8 kernel of 0.2.6), =1 as the default.
if (g_pa_imma < 0) {
const char* v = std::getenv("STRATA_PROMPT_ATTN_IMMA");
- g_pa_imma = v != nullptr && v[0] == '1' ? 1 : 0;
+ g_pa_imma = v == nullptr ? 2 : (v[0] == '1' ? 1 : 0); // 2: the default (on where the kernel can run)
}
- if (g_pa_imma == 1 && !turing) return launch_i8v3(q, pools, ids, steps, cap, s, attn, n_q, st);
+ if (!turing && (g_pa_imma == 1 || (g_pa_imma == 2 && strata::prompt_attn_imma_default(false, 80)))) return launch_i8v3(q, pools, ids, steps, cap, s, attn, n_q, st);
return launch_i8(q, pools, ids, steps, cap, s, attn, n_q, st);
}
if (!pools.k_pool || !pools.v_pool) return false;
diff --git a/src/kernels/fused_gr_bench.cpp b/src/kernels/fused_gr_bench.cpp
index 0ac95eaf3..677696cf1 100644
--- a/src/kernels/fused_gr_bench.cpp
+++ b/src/kernels/fused_gr_bench.cpp
@@ -55,6 +55,12 @@ static std::vector<uint8_t> q8_0_of_bf16(const std::vector<uint16_t>& w, size_t
}
int main(int argc, char** argv) {
+ // the bench's contract is the default kernels' bitwise hashes; the split read (a default on a 12.0 card since 0.2.7) is another summation order
+#if defined(_WIN32)
+ if (std::getenv("STRATA_GR_V3") == nullptr) _putenv_s("STRATA_GR_V3", "0");
+#else
+ if (std::getenv("STRATA_GR_V3") == nullptr) setenv("STRATA_GR_V3", "0", 1);
+#endif
setvbuf(stdout, nullptr, _IONBF, 0);
const int iters = argc > 1 ? std::atoi(argv[1]) : 500;
const int t_lo = argc > 2 ? std::atoi(argv[2]) : 1, t_hi = argc > 3 ? std::atoi(argv[3]) : 6;
diff --git a/src/kernels/gr_parity.cpp b/src/kernels/gr_parity.cpp
index 34d7510bf..22da807be 100644
--- a/src/kernels/gr_parity.cpp
+++ b/src/kernels/gr_parity.cpp
@@ -480,6 +480,12 @@ int fused_multi_lds_parity(const float* d_norm, const uint16_t* d_down, const ui
} // namespace
int main(int argc, char** argv) {
+ // this test compares the default kernels bit for bit; the split read (a default on a 12.0 card since 0.2.7) is STRATA_GR_V3=1 below
+#if defined(_WIN32)
+ if (std::getenv("STRATA_GR_V3") == nullptr) _putenv_s("STRATA_GR_V3", "0");
+#else
+ if (std::getenv("STRATA_GR_V3") == nullptr) setenv("STRATA_GR_V3", "0", 1);
+#endif
bool selftest = false;
for (int i = 1; i < argc; ++i) {
if (std::string(argv[i]) == "--selftest") selftest = true;
diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cpp
index 1f9bfee73..6fb5a1d81 100644
--- a/tests/dsv41/experts_rule_test.cpp
+++ b/tests/dsv41/experts_rule_test.cpp
@@ -38,6 +38,15 @@ int main() {
CHECK_EQ(experts_per_word(-1, 24 * GiB, 6, false), 6); // a discrete 24 GB card
CHECK_EQ(experts_per_word(-1, 12 * GiB, 6, false), 5); // a discrete 12 GB card
CHECK_EQ(experts_per_word(6, 91 * GiB, 6, true), 6); // --experts 6 wins on the unified-memory GPU too
+ // two or more RDNA4 cards that hold at least half of the experts (4 x R9700, 128 GiB in all): faster mode by default (the 5th argument)
+ CHECK_EQ(experts_per_word(-1, 4 * 32 * GiB, 6, false, true), 5);
+ CHECK_EQ(experts_per_word(-1, 3 * 32 * GiB, 6, false, true), 5);
+ CHECK_EQ(experts_per_word(-1, 4 * 32 * GiB, 6, false, false), 6); // the same cards when they hold under half (or are not RDNA4): the model's 6
+ CHECK_EQ(experts_per_word(6, 4 * 32 * GiB, 6, false, true), 6); // --experts 6 (the app's / Terminal's "Full quality") wins
+ CHECK_EQ(experts_per_word(4, 4 * 32 * GiB, 6, false, true), 4); // so does any other --experts N
+ CHECK_EQ(experts_per_word(5, 4 * 32 * GiB, 6, false, false), 5);
+ CHECK_EQ(experts_per_word(-1, 4 * 32 * GiB, 1, false, true), 1); // a model of one expert a word has nothing to drop
+ CHECK_EQ(experts_test_override(experts_per_word(-1, 4 * 32 * GiB, 6, false, true), 6), 5);
CHECK_EQ(experts_per_word(-1, 91 * GiB, 4, true), 4); // never raised
// unknown memory (0): the model's count, never a silent cut
CHECK_EQ(experts_per_word(-1, 0, 6), 6);