Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
CUDA Plan opt-ins become defaults where measured: STRATA_GR_V3 on a discrete 12.0 card above 13 GiB, STRATA_PROMPT_ATTN_IMMA on CUDA sm_80+ (env =0 turns each off); DeepSeek Faster mode (5 experts) by default on 2+ RDNA4 cards that hold half the experts; the route-prior/GR_V3/IMMA rules are pure functions with tests; gr_parity and fused_gr_bench pin STRATA_GR_V3=0 for their bitwise contract
Niko122154d9363c972c
diff --git a/include/strata/core/sm120_series.hpp b/include/strata/core/sm120_series.hppindex 9059d312e..b1cc11925 100644--- a/include/strata/core/sm120_series.hpp+++ b/include/strata/core/sm120_series.hpp@@ -42,4 +42,27 @@ inline bool is_sm120(int major, int minor) { return cc_major_of(major) == 12 &&/// the default of a bitwise switch of the set: on for a 12.0 card unless STRATA_SM120_PERF=0inline bool sm120_default(int major, int minor) { return is_sm120(major, minor) && sm120_series(); }+// ---- 0.2.7 defaults of the CUDA speed work (pure rules; the engine reads the device and calls these; tests in arch_defaults_test)+// `hip`: a HIP build, where a gfx1200 / gfx1201 reports compute capability 12.0 as well and none of these is measured or wanted.+// `total_bytes`: the card's memory (cudaDeviceProp::totalGlobalMem); `integrated`: a GPU that shares the PC's memory.+constexpr double kSmallCardGiB = 13.0; // "a 12 GB card": 12 GB cards report 11.6 - 12.0 GiB, 16 GB ones 15.6 - 16.0++/// STRATA_ROUTE_PRIOR when the user set nothing: 2.0 (residency-biased routing, changes which experts run; owner-approved for 12 GB cards) on a+/// discrete CUDA card of 13 GiB or less, 0 everywhere else. Measured on an RTX 3060 12 GB only.+inline float route_prior_default(bool hip, bool integrated, double total_bytes) {+ return !hip && !integrated && total_bytes > 0.0 && total_bytes / 1073741824.0 <= kSmallCardGiB ? 2.0f : 0.0f;+}++/// STRATA_GR_V3 (the stream-split hyper-connection read, another summation order) when the user set nothing: ON for a discrete compute capability+/// 12.0 card (STRATA_SM120_PERF=0 turns the set off) with more than 13 GiB, CUDA builds only. Measured: RTX PRO 5000 72 GB +1.2% decode clean,+/// -3.0% / -5.9% on 2x RTX A4000 (sm_86), +1.0% / -3.4% on an RTX 3060 (sm_86): so not those; a 12 GB sm_120 card is unmeasured.+inline bool gr_v3_default(bool hip, int major, int minor, bool integrated, double total_bytes) {+ return !hip && !integrated && sm120_default(major, minor) && total_bytes / 1073741824.0 > kSmallCardGiB;+}++/// STRATA_PROMPT_ATTN_IMMA (the prompt attention on int8 tensor cores, another summation order) when the user set nothing: ON for every CUDA card of+/// compute capability 8.0 or newer (the int8 m16n8k32 MMA), not Turing / Volta / Pascal, not HIP. `cc` = 10 * major + minor as the engine+/// resolves it (an emulated or forced Turing counts as 75). Measured on sm_120 (RTX PRO 5000) and sm_86 (2x RTX A4000): +1.1 .. +1.8% prompt.+inline bool prompt_attn_imma_default(bool hip, int cc) { return !hip && cc >= 80; }+} // namespace stratadiff --git a/src/core/arch_defaults_test.cpp b/src/core/arch_defaults_test.cppindex 459fdaa49..9723b2b78 100644--- a/src/core/arch_defaults_test.cpp+++ b/src/core/arch_defaults_test.cpp@@ -1,6 +1,7 @@// src/core/arch_defaults_test.cpp - the per-architecture default table (no GPU needed).#include "strata/core/arch_defaults.hpp"#include "strata/core/gfx12_platform_defaults.hpp"+#include "strata/core/sm120_series.hpp"#include <cstdio>#include <cstdlib>@@ -226,5 +227,46 @@ int main() {unset("STRATA_UMA_REGISTER");// both platforms: no Windows guard on the iGPU set (the owner wants it there too; untested on Windows is only said in the log)std::printf(fails ? "arch_defaults_test: %d FAILED\n" : "arch_defaults_test: OK\n", fails);+ // ---- 0.2.7 CUDA defaults (sm120_series.hpp): which cards get which, and which never do+ {+ const double GiB = 1073741824.0;+ using namespace strata;+ // residency-biased routing: 2.0 on a discrete CUDA card of 13 GiB or less, nothing else+ CHECK(route_prior_default(false, false, 11.9 * GiB) == 2.0f); // RTX 3060 / 5070 12 GB+ CHECK(route_prior_default(false, false, 12.0 * GiB) == 2.0f);+ CHECK(route_prior_default(false, false, 8.0 * GiB) == 2.0f); // an 8 GB card+ CHECK(route_prior_default(false, false, 13.0 * GiB) == 2.0f); // the boundary is inclusive+ CHECK(route_prior_default(false, false, 13.5 * GiB) == 0.0f);+ CHECK(route_prior_default(false, false, 15.7 * GiB) == 0.0f); // 16 GB card: z440's A4000, P100+ CHECK(route_prior_default(false, false, 72.0 * GiB) == 0.0f); // RTX PRO 5000+ CHECK(route_prior_default(true, false, 12.0 * GiB) == 0.0f); // a HIP build: a 12 GB RX 6700 XT / 7700 XT+ CHECK(route_prior_default(true, true, 12.0 * GiB) == 0.0f); // Strix Halo with a small carve-out+ CHECK(route_prior_default(false, true, 8.0 * GiB) == 0.0f); // an integrated GPU on CUDA (Jetson-like)+ CHECK(route_prior_default(false, false, 0.0) == 0.0f); // unknown memory+ // the stream-split hyper-connection read: a discrete 12.0 card above 13 GiB (RTX PRO 5000 72 GB, RTX 5090 32 GB), nothing else+ CHECK(gr_v3_default(false, 12, 0, false, 72.0 * GiB));+ CHECK(gr_v3_default(false, 12, 0, false, 31.4 * GiB)); // RTX 5090+ CHECK(!gr_v3_default(false, 12, 0, false, 11.9 * GiB)); // RTX 5070 12 GB: streams most experts, unmeasured+ CHECK(gr_v3_default(false, 12, 0, false, 15.6 * GiB)); // a 16 GB sm_120 card (RTX 5060 Ti / 5070 Ti): above 13 GiB, on (unmeasured size, same arch)+ CHECK(!gr_v3_default(false, 8, 6, false, 15.7 * GiB)); // 2x RTX A4000 (sm_86): measured -3.0% / -5.9%+ CHECK(!gr_v3_default(false, 8, 6, false, 72.0 * GiB)); // any sm_86+ CHECK(!gr_v3_default(false, 8, 9, false, 24.0 * GiB)); // sm_89+ CHECK(!gr_v3_default(false, 12, 1, false, 128.0 * GiB)); // sm_121 (DGX Spark class): not 12.0+ CHECK(!gr_v3_default(false, 12, 0, true, 128.0 * GiB)); // integrated+ CHECK(!gr_v3_default(true, 12, 0, false, 32.0 * GiB)); // a HIP build: gfx1201 reports 12.0 too+ CHECK(!gr_v3_default(false, 6, 0, false, 16.0 * GiB)); // P100+ }+ {+ using namespace strata;+ // the prompt attention on int8 tensor cores: every CUDA card from sm_80 on, not Turing / Volta / Pascal, not HIP+ CHECK(prompt_attn_imma_default(false, 120)); // RTX PRO 5000 / RTX 50: measured +1.6-1.8%+ CHECK(prompt_attn_imma_default(false, 86)); // 2x RTX A4000 / RTX 3060: measured +1.1-1.3%+ CHECK(prompt_attn_imma_default(false, 80));+ CHECK(prompt_attn_imma_default(false, 89));+ CHECK(!prompt_attn_imma_default(false, 75)); // Turing (also a forced STRATA_QSA_WARP=1 card)+ CHECK(!prompt_attn_imma_default(false, 70)); // Volta+ CHECK(!prompt_attn_imma_default(false, 60)); // Pascal (P100)+ CHECK(!prompt_attn_imma_default(true, 120)); // HIP+ }return fails ? 1 : 0;}diff --git a/src/core/verify.cpp b/src/core/verify.cppindex b9d4eeded..f99dc9e8b 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -225,7 +225,7 @@ inline float g_route_prior() {cudaDeviceProp p;const bool ok = cudaGetDeviceProperties(&p, dev) == cudaSuccess;if (!ok) cudaGetLastError();- per_dev[dev] = 1.0f + (ok && !p.integrated && (double) p.totalGlobalMem / 1073741824.0 <= 13.0 ? 2.0f : 0.0f);+ per_dev[dev] = 1.0f + (ok ? strata::route_prior_default(false, p.integrated != 0, (double) p.totalGlobalMem) : 0.0f);}return per_dev[dev] - 1.0f;#endifdiff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hppindex 140b3df4a..ae872def0 100644--- a/src/dsv41/ds4_experts_rule.hpp+++ b/src/dsv41/ds4_experts_rule.hpp@@ -18,10 +18,13 @@ constexpr int kFasterModeExperts = 5;// requested: --experts (-1: not given, the rule decides); vram_bytes: the total memory of the cards the engine uses;// model_k: the experts a word takes in the model; integrated: a GPU that shares the PC's RAM (Strix Halo: part of the experts live only on// the SSD, each expert dropped is less read - measured 5 against 6 on a Radeon 8065S: decode +11..13%, prompts +5..7%). The result is in 1 .. model_k.-inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k, bool integrated = false) {+// multi_rdna4: two or more RDNA4 cards (gfx1200 / gfx1201: R9700, RX 9000) that together hold at least half of the routed experts - the 4 x R9700 box+// the Marlin stream measured: 5 against 6 experts, 2.25 bpw, 8K prompt 1339 -> 1419 ppts (+6%) and decode 78.1 -> 81.2 tok/s (+4%), reference KL+// 0.10987 against 0.10833 and top-1 87.59% against 87.53% (the gate is 0.006). Faster mode there too, by default.+inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k, bool integrated = false, bool multi_rdna4 = false) {if (model_k <= 1) return model_k;int n = requested;- if (n < 1) n = integrated || (vram_bytes != 0 && vram_bytes <= kFasterModeVramMax) ? kFasterModeExperts : model_k;+ if (n < 1) n = integrated || multi_rdna4 || (vram_bytes != 0 && vram_bytes <= kFasterModeVramMax) ? kFasterModeExperts : model_k;if (n > model_k) n = model_k;return n < 1 ? 1 : n;}diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex e36ed7673..9c3596441 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -2132,11 +2132,13 @@ struct Model::Impl {cudaDeviceProp ip{};if (cudaGetDeviceProperties(&ip, dv.id) == cudaSuccess && ip.integrated) integrated = true;}- const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp, integrated), l.k_exp);+ // two or more RDNA4 cards holding at least half of the experts (the 4 x R9700 class): Faster by default too+ const bool multi_rdna4 = rdna4() && devs.size() >= 2 && gpu_expert_share() >= 0.5;+ const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp, integrated, multi_rdna4), l.k_exp);experts_used = n;if (n < l.k_exp)std::fprintf(stderr, "ds4: experts per word: %d (%s; full quality: --experts %d)\n", n,- o.experts >= 1 ? "as asked" : "faster mode", l.k_exp);+ o.experts >= 1 ? "as asked" : multi_rdna4 ? "faster mode: the cards hold most of the experts" : "faster mode", l.k_exp);else if (o.experts >= 1)std::fprintf(stderr, "ds4: experts per word: %d (full quality)\n", n);}diff --git a/src/kernels/cuda/fused_gr.cu b/src/kernels/cuda/fused_gr.cuindex 73a9089b7..ad1fcbd34 100644--- a/src/kernels/cuda/fused_gr.cu+++ b/src/kernels/cuda/fused_gr.cu@@ -2302,6 +2302,32 @@ int g_gr_fast = -1;#endifvoid fused_gr_set_fast(int on) { g_gr_fast = on; }+// STRATA_GR_V3: the stream-split hyper-connection read (gr_down_v3_kernel / gr_up_v3_kernel). Another summation order than the default+// read (~1e-7; a greedy tie can flip, so the answers are not byte-identical to 0.2.6's). 0.2.7: ON by default on a compute capability 12.0+// card (RTX 50 / RTX PRO Blackwell: sm120_series.hpp, STRATA_SM120_PERF=0 turns the set off) with more than 13 GiB of memory, discrete;+// measured on the RTX PRO 5000 72 GB with every expert resident: code decode 245.1 -> 248.1 tok/s (+1.2%, identical tokens), a story prompt+// +6.4% (tokens changed, not a claim), held-out document prompt +4.3%, with STRATA_PROMPT_ATTN_IMMA=1 +1.5% decode / +1.6-1.8% prompt together,+// 5 pairs. NOT a default elsewhere: it measured -3.0% / -5.9% on 2x RTX A4000 (sm_86) and +1.0% / -3.4% on an RTX 3060 (sm_86), and a 12 GB+// sm_120 card (RTX 5070: streams most experts, the read is a small share of its window) is unmeasured. STRATA_GR_V3=0 / 1 forces it off / on+// on any card; never in a HIP build (gfx12 reports compute capability 12.0 too). Decided per card, first asked in fused_gr_check (before any+// graph capture).+static bool gr_v3_default_on(int dev) {+#if defined(__HIPCC__)+ (void) dev;+ return false;+#else+ static int per_dev[64]; // 0 unknown, 1 no, 2 yes+ if (dev < 0 || dev >= 64) return false;+ if (!per_dev[dev]) {+ cudaDeviceProp p;+ const bool ok = cudaGetDeviceProperties(&p, dev) == cudaSuccess;+ if (!ok) cudaGetLastError();+ per_dev[dev] = ok && strata::gr_v3_default(false, p.major, p.minor, p.integrated != 0, (double) p.totalGlobalMem) ? 2 : 1;+ }+ return per_dev[dev] == 2;+#endif+}+bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream, unsigned long long* stamp_buf,int stamp_i0) {if (n_tok < 1 || n_tok > kFusedGrMaxT || xn_scratch == nullptr) {@@ -2337,12 +2363,13 @@ bool fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi}return q8;}- // STRATA_GR_V3=1: the two-kernel read above (another summation order - opt-in)- static const bool v3 = [] { const char* v = std::getenv("STRATA_GR_V3"); return v != nullptr && std::atoi(v) != 0; }();+ // the two-kernel read above (another summation order): STRATA_GR_V3=0 / 1, unset = gr_v3_default_on (a 12.0 card with > 13 GiB)+ static const int v3_env = [] { const char* v = std::getenv("STRATA_GR_V3"); return v != nullptr ? (std::atoi(v) != 0 ? 1 : 0) : -1; }();static int split3[64] = {}; // per device: 0 = not decided yet, 1 / 2 = column halves S, -1 = does not fitint dev3 = 0;+ cudaGetDevice(&dev3);+ const bool v3 = v3_env >= 0 ? v3_env == 1 : gr_v3_default_on(dev3);if (v3) {- cudaGetDevice(&dev3);if (dev3 >= 0 && dev3 < 64 && split3[dev3] == 0) {int optin = 0;cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev3);@@ -2653,6 +2680,7 @@ int fused_gr_variant() {void fused_gr_check() {int dev = 0;cudaGetDevice(&dev);+ (void) gr_v3_default_on(dev); // decided here, before any graph capture asksif (dev < 0 || dev >= 64 || g_variant[dev].load() > 0) return;const int want = env_variant();if (want == kHcPlain) {diff --git a/src/kernels/cuda/qsa_prompt_attn.cu b/src/kernels/cuda/qsa_prompt_attn.cuindex 3bdc4ffc9..f2b2d0a06 100644--- a/src/kernels/cuda/qsa_prompt_attn.cu+++ b/src/kernels/cuda/qsa_prompt_attn.cu@@ -1,5 +1,6 @@// src/kernels/cuda/qsa_prompt_attn.cu - see include/strata/kernels/qsa_prompt_attn.hpp.#include "strata/core/emulate.hpp"+#include "strata/core/sm120_series.hpp"#include "strata/core/sm75_series.hpp"#include "strata/core/geometry.hpp"#include "strata/core/gfx12_platform_defaults.hpp"@@ -2418,12 +2419,16 @@ bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int3if (pf_env == 1 || (pf_env < 0 && t75 && strata::sm75_series())) return launch<1, true>(q, pools, ids, steps, cap, s, attn, n_q, st);return launch<1>(q, pools, ids, steps, cap, s, attn, n_q, st);}- // STRATA_PROMPT_ATTN_IMMA=1 (opt-in, sm_80+): both products on INT8 tensor cores (prompt_attn_i8v3_kernel)+ // STRATA_PROMPT_ATTN_IMMA (sm_80+): both products on INT8 tensor cores (prompt_attn_i8v3_kernel). ON by default since 0.2.7 on every+ // card that has them (Ampere and newer, CUDA builds; not Turing, Volta or Pascal, which have no int8 m16n8k32 MMA this kernel needs):+ // +1.6% / +1.8% prompt at 8K / 16K on the RTX PRO 5000 (sm_120) and +1.1% / +1.3% at 8K / 16K on 2x RTX A4000 (sm_86), decode unchanged,+ // 5 pairs each; another summation order (~1e-7), so a near-tie can flip a prompt's first token. Sm_80, 8.9, 9.0 and the datacentre+ // Blackwells run the same kernel unmeasured. STRATA_PROMPT_ATTN_IMMA=0 is the off switch (the v2 i8 kernel of 0.2.6), =1 as the default.if (g_pa_imma < 0) {const char* v = std::getenv("STRATA_PROMPT_ATTN_IMMA");- g_pa_imma = v != nullptr && v[0] == '1' ? 1 : 0;+ g_pa_imma = v == nullptr ? 2 : (v[0] == '1' ? 1 : 0); // 2: the default (on where the kernel can run)}- if (g_pa_imma == 1 && !turing) return launch_i8v3(q, pools, ids, steps, cap, s, attn, n_q, st);+ if (!turing && (g_pa_imma == 1 || (g_pa_imma == 2 && strata::prompt_attn_imma_default(false, 80)))) return launch_i8v3(q, pools, ids, steps, cap, s, attn, n_q, st);return launch_i8(q, pools, ids, steps, cap, s, attn, n_q, st);}if (!pools.k_pool || !pools.v_pool) return false;diff --git a/src/kernels/fused_gr_bench.cpp b/src/kernels/fused_gr_bench.cppindex 0ac95eaf3..677696cf1 100644--- a/src/kernels/fused_gr_bench.cpp+++ b/src/kernels/fused_gr_bench.cpp@@ -55,6 +55,12 @@ static std::vector<uint8_t> q8_0_of_bf16(const std::vector<uint16_t>& w, size_t}int main(int argc, char** argv) {+ // the bench's contract is the default kernels' bitwise hashes; the split read (a default on a 12.0 card since 0.2.7) is another summation order+#if defined(_WIN32)+ if (std::getenv("STRATA_GR_V3") == nullptr) _putenv_s("STRATA_GR_V3", "0");+#else+ if (std::getenv("STRATA_GR_V3") == nullptr) setenv("STRATA_GR_V3", "0", 1);+#endifsetvbuf(stdout, nullptr, _IONBF, 0);const int iters = argc > 1 ? std::atoi(argv[1]) : 500;const int t_lo = argc > 2 ? std::atoi(argv[2]) : 1, t_hi = argc > 3 ? std::atoi(argv[3]) : 6;diff --git a/src/kernels/gr_parity.cpp b/src/kernels/gr_parity.cppindex 34d7510bf..22da807be 100644--- a/src/kernels/gr_parity.cpp+++ b/src/kernels/gr_parity.cpp@@ -480,6 +480,12 @@ int fused_multi_lds_parity(const float* d_norm, const uint16_t* d_down, const ui} // namespaceint main(int argc, char** argv) {+ // this test compares the default kernels bit for bit; the split read (a default on a 12.0 card since 0.2.7) is STRATA_GR_V3=1 below+#if defined(_WIN32)+ if (std::getenv("STRATA_GR_V3") == nullptr) _putenv_s("STRATA_GR_V3", "0");+#else+ if (std::getenv("STRATA_GR_V3") == nullptr) setenv("STRATA_GR_V3", "0", 1);+#endifbool selftest = false;for (int i = 1; i < argc; ++i) {if (std::string(argv[i]) == "--selftest") selftest = true;diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cppindex 1f9bfee73..6fb5a1d81 100644--- a/tests/dsv41/experts_rule_test.cpp+++ b/tests/dsv41/experts_rule_test.cpp@@ -38,6 +38,15 @@ int main() {CHECK_EQ(experts_per_word(-1, 24 * GiB, 6, false), 6); // a discrete 24 GB cardCHECK_EQ(experts_per_word(-1, 12 * GiB, 6, false), 5); // a discrete 12 GB cardCHECK_EQ(experts_per_word(6, 91 * GiB, 6, true), 6); // --experts 6 wins on the unified-memory GPU too+ // two or more RDNA4 cards that hold at least half of the experts (4 x R9700, 128 GiB in all): faster mode by default (the 5th argument)+ CHECK_EQ(experts_per_word(-1, 4 * 32 * GiB, 6, false, true), 5);+ CHECK_EQ(experts_per_word(-1, 3 * 32 * GiB, 6, false, true), 5);+ CHECK_EQ(experts_per_word(-1, 4 * 32 * GiB, 6, false, false), 6); // the same cards when they hold under half (or are not RDNA4): the model's 6+ CHECK_EQ(experts_per_word(6, 4 * 32 * GiB, 6, false, true), 6); // --experts 6 (the app's / Terminal's "Full quality") wins+ CHECK_EQ(experts_per_word(4, 4 * 32 * GiB, 6, false, true), 4); // so does any other --experts N+ CHECK_EQ(experts_per_word(5, 4 * 32 * GiB, 6, false, false), 5);+ CHECK_EQ(experts_per_word(-1, 4 * 32 * GiB, 1, false, true), 1); // a model of one expert a word has nothing to drop+ CHECK_EQ(experts_test_override(experts_per_word(-1, 4 * 32 * GiB, 6, false, true), 6), 5);CHECK_EQ(experts_per_word(-1, 91 * GiB, 4, true), 4); // never raised// unknown memory (0): the model's count, never a silent cutCHECK_EQ(experts_per_word(-1, 0, 6), 6);