Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
layer-major prefill: on by default for a CUDA layer split, STRATA_PREFILL_LAYER_MAJOR=0 and STRATA_DEFAULTS=026 turn it off (#3976)
Niko122137e4832ab3e0
diff --git a/include/strata/core/sm120_series.hpp b/include/strata/core/sm120_series.hppindex e116e0ff3..3468def3d 100644--- a/include/strata/core/sm120_series.hpp+++ b/include/strata/core/sm120_series.hpp@@ -98,4 +98,19 @@ inline bool mmvq_il_89_resolve(const char* env, bool hip, int cc, bool v026 = fa/// capability 12.0 card (STRATA_SM120_PERF=0 turns the set off), CUDA builds only.inline bool gr_up_wide_default(bool hip, int major, int minor, bool v026 = false) { return !v026 && !hip && sm120_default(major, minor); }+/// STRATA_PREFILL_LAYER_MAJOR (#3976: the layer-major prompt walk, which reads each layer's streamed experts once per super-chunk instead of once+/// per chunk; another summation order of the routed experts, so KL-checked, not bitwise) when the user set nothing: ON for a CUDA layer split+/// (`stages` = the number of GPUs the prompt path runs on, 2 or more), OFF on one GPU, in a HIP build and under STRATA_DEFAULTS=026. Measured on+/// 2x RTX A4000 (sm_86) with Qwen3.8-Flash-Next IQ3_XXS and streamed experts only. The engine adds its own per-prompt declines (image prompts,+/// DFlash2 taps, KVA, a batch read that yields, a peer GPU, no VRAM for a super-chunk): those read the chunk-major path as before.+inline bool prefill_layer_major_default(bool hip, int stages, bool v026 = false) { return !v026 && !hip && stages >= 2; }++/// The answer for STRATA_PREFILL_LAYER_MAJOR given the variable's text (nullptr = unset): any text that starts a number other than 0 is on, "0" (or any+/// text that reads as 0) is off, both win over everything; unset or empty is the rule above. An explicit "1" is honoured on one GPU too (the+/// opt-in of the first ports); a HIP build still declines at run time.+inline bool prefill_layer_major_resolve(const char* env, bool hip, int stages, bool v026 = false) {+ if (env != nullptr && env[0] != '\0') return std::atoi(env) != 0;+ return prefill_layer_major_default(hip, stages, v026);+}+} // namespace stratadiff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hppindex da26dcc4c..549aff380 100644--- a/include/strata/prefill/prefill.hpp+++ b/include/strata/prefill/prefill.hpp@@ -284,6 +284,7 @@ private:bool run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err);void lm_release();bool lm_mid_ = false;+ bool lm_by_default_ = false; ///< the walk is on because nothing was set (a wider VRAM margin, the measured super-chunk size)// Stage-1 pipeline: intermediate stages return after handing their chunk to// the direct successor. The public run() drains the chain once at prompt end.bool run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err);diff --git a/src/core/arch_defaults_test.cpp b/src/core/arch_defaults_test.cppindex 290f183d6..e13e76b1d 100644--- a/src/core/arch_defaults_test.cpp+++ b/src/core/arch_defaults_test.cpp@@ -396,5 +396,30 @@ int main() {CHECK(!prompt_attn_imma_default(false, 60)); // Pascal (P100)CHECK(!prompt_attn_imma_default(true, 120)); // HIP}+ {+ using namespace strata;+ // the layer-major prompt walk (#3976): on by default for a CUDA layer split only+ CHECK(prefill_layer_major_default(false, 2)); // 2x RTX A4000 / A6000: measured 5x time to first token+ CHECK(prefill_layer_major_default(false, 4));+ CHECK(!prefill_layer_major_default(false, 1)); // one GPU: opt-in only+ CHECK(!prefill_layer_major_default(false, 0));+ CHECK(!prefill_layer_major_default(true, 2)); // HIP builds+ CHECK(!prefill_layer_major_default(false, 2, true)); // STRATA_DEFAULTS=026+ CHECK(!prefill_layer_major_default(true, 1, true));+ // the variable wins in both directions, whatever the rule says+ CHECK(!prefill_layer_major_resolve("0", false, 2)); // the off switch+ CHECK(prefill_layer_major_resolve("1", false, 1)); // the opt-in of the first ports: one GPU too+ CHECK(prefill_layer_major_resolve("1", false, 2, true)); // explicit beats STRATA_DEFAULTS=026+ CHECK(!prefill_layer_major_resolve("0", false, 2, false));+ CHECK(!prefill_layer_major_resolve("0", true, 2));+ CHECK(prefill_layer_major_resolve("2", false, 1)); // any non-zero number is on+ CHECK(!prefill_layer_major_resolve("off", false, 2)); // text that reads as 0 is off (atoi), as before this rule existed+ // unset or empty: the rule+ CHECK(prefill_layer_major_resolve(nullptr, false, 2));+ CHECK(prefill_layer_major_resolve("", false, 2));+ CHECK(!prefill_layer_major_resolve(nullptr, false, 1));+ CHECK(!prefill_layer_major_resolve(nullptr, false, 2, true));+ CHECK(!prefill_layer_major_resolve(nullptr, true, 2));+ }return fails ? 1 : 0;}diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex 2e424ff0e..16a609a5d 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -49,6 +49,8 @@#include "strata/prefill/moe_mmq.hpp"#include "strata/core/peer_experts.hpp"#include "strata/core/expert_nibble.hpp"+#include "strata/core/sm120_series.hpp"+#include "strata/core/defaults_026.hpp"#include "strata/prefill/kernels.hpp"#include "strata/prefill/kva.hpp" // STRATA_KVA: the optional approximate-prefill plan (research, off by default)@@ -5874,10 +5876,11 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri// other row groupings. on_chunk reports every sub-chunk in order from the final pass; while it reports one before// the super-chunk's end, layer_major_mid() is true (the session state is the super-chunk's end: no checkpoint).namespace {-bool lm_env() {- static const bool v = [] { const char* e = std::getenv("STRATA_PREFILL_LAYER_MAJOR"); return e && std::atoi(e) != 0; }();- return v;+const char* lm_env_text() { // STRATA_PREFILL_LAYER_MAJOR as set (nullptr: unset); the rule for unset is strata::prefill_layer_major_default+ static const char* const e = std::getenv("STRATA_PREFILL_LAYER_MAJOR");+ return e;}+bool lm_env_explicit() { const char* e = lm_env_text(); return e != nullptr && e[0] != '\0'; }bool lm_even_env() { // even super-chunks (STRATA_PREFILL_LM_EVEN=0: the largest that fits, then the rest)static const bool v = [] { const char* e = std::getenv("STRATA_PREFILL_LM_EVEN"); return !(e && e[0] == '0'); }();@@ -5897,14 +5900,18 @@ int lm_r_env() { // 0 auto, 1 vram, 2 host}();return v;}-int64_t lm_margin_bytes() { // VRAM left free while a prompt runs (the verify windows, the driver)- static const int64_t v = [] {+// VRAM left free while a prompt runs (the verify windows, the driver): 96 MiB when STRATA_PREFILL_LAYER_MAJOR=1 asked for the walk, 512 MiB when+// it is on by default (a user who set nothing gets the wider margin)+int64_t lm_margin_bytes(bool by_default) {+ static const int64_t explicit_v = [] {const char* e = std::getenv("STRATA_PREFILL_LM_MARGIN_MIB");- return (int64_t) (e ? std::atoll(e) : 96) << 20;+ return e ? (int64_t) std::atoll(e) << 20 : (int64_t) -1;}();- return v;+ return explicit_v >= 0 ? explicit_v : (int64_t) (by_default ? 512 : 96) << 20;}constexpr int64_t kLmDefaultTokens = 8192;+// the super-chunk target when the walk is on by default and STRATA_PREFILL_LM_TOKENS is not set: the size measured (2x RTX A4000, 32K prompts)+constexpr int64_t kLmDefaultOnTokens = 32768;// the device tables of one layer, in int32s: row_tok (rows) | bounds (2 E) | u_tok (rows) | u_off (rows + E) |// u_rows (rows) | u_w (rows), E = the most entries a layer can split intoint64_t lm_entries_max(int64_t rows, int64_t G) { return NE + rows / std::max<int64_t>(G, 1) + 2; }@@ -5991,7 +5998,7 @@ int64_t Prefill::lm_prepare(int64_t n, std::string& why, int64_t cap) {// (an explicit STRATA_PREFILL_LM_TOKENS may be one chunk: the A/B against the default path, experts aside)const int64_t min_s = lm_tokens_env() > 0 ? m.T : 2 * m.T;if (n < min_s) { why = "a short prompt"; return 0; }- const int64_t want = lm_tokens_env() > 0 ? lm_tokens_env() : kLmDefaultTokens;+ const int64_t want = lm_tokens_env() > 0 ? lm_tokens_env() : (lm_by_default_ ? kLmDefaultOnTokens : kLmDefaultTokens);int64_t S = std::max<int64_t>(min_s, want / m.T * m.T);S = std::min<int64_t>(S, (n + m.T - 1) / m.T * m.T);if (cap > 0) {@@ -6006,7 +6013,7 @@ int64_t Prefill::lm_prepare(int64_t n, std::string& why, int64_t cap) {};size_t free_b = 0, total_b = 0;cudaMemGetInfo(&free_b, &total_b);- const int64_t room = (int64_t) free_b - lm_margin_bytes();+ const int64_t room = (int64_t) free_b - lm_margin_bytes(lm_by_default_);m.lm.free_mib = (int64_t) (free_b >> 20);auto fit = [&](bool r_dev) { // the largest super-chunk (a multiple of the chunk) that fits `room`int64_t s = S;@@ -7073,8 +7080,27 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& ereturn ok;}#endif- if (lm_env() && hand_in_ == nullptr && impl_) {+#if defined(STRATA_USE_HIP) || defined(STRATA_HIP_GFX906)+ constexpr bool lm_hip = true;+#else+ constexpr bool lm_hip = false;+#endif+ // layer-major prefill: STRATA_PREFILL_LAYER_MAJOR=1 / 0 win; unset, it is on for a CUDA layer split (sm120_series.hpp+ // prefill_layer_major_default; STRATA_DEFAULTS=026 and a HIP build leave it off)+ if (hand_in_ == nullptr && impl_ &&+ strata::prefill_layer_major_resolve(lm_env_text(), lm_hip, stage_count(), strata::qwen_defaults_026())) {+ const bool by_default = !lm_env_explicit();+ for (Prefill* p = this; p != nullptr; p = p->next_) p->lm_by_default_ = by_default;std::string why;+ if (by_default) {+ static bool noted = false;+ if (!noted) {+ noted = true;+ std::fprintf(stderr, "strata prefill: layer-major prefill is on for this layer split (each layer's experts are read once per "+ "super-chunk of up to %lld tokens, not once per chunk; STRATA_PREFILL_LAYER_MAJOR=0 or STRATA_DEFAULTS=026 "+ "turns it off)\n", (long long) kLmDefaultOnTokens);+ }+ }if (lm_prepare_chain(n, why) > 0) {bool ok = run_layer_major(tokens, n, pos0, err);std::string drain_err; // a layer split: the later stages' last super-chunk@@ -7086,8 +7112,8 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& elm_release_chain();return ok;}- static std::string said; // once per reason (a short prompt is the common one)- if (why != said) {+ static std::string said; // once per reason (a short prompt is the common one; not worth a line when nobody asked for the walk)+ if (why != said && !(by_default && why == "a short prompt")) {said = why;std::fprintf(stderr, "strata prefill: layer-major off for this prompt: %s\n", why.c_str());}