Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

layer-major prefill: on by default for a CUDA layer split, STRATA_PREFILL_LAYER_MAJOR=0 and STRATA_DEFAULTS=026 turn it off (#3976)

Niko122137e4832ab3e0
diff --git a/include/strata/core/sm120_series.hpp b/include/strata/core/sm120_series.hpp
index e116e0ff3..3468def3d 100644
--- a/include/strata/core/sm120_series.hpp
+++ b/include/strata/core/sm120_series.hpp
@@ -98,4 +98,19 @@ inline bool mmvq_il_89_resolve(const char* env, bool hip, int cc, bool v026 = fa
/// capability 12.0 card (STRATA_SM120_PERF=0 turns the set off), CUDA builds only.
inline bool gr_up_wide_default(bool hip, int major, int minor, bool v026 = false) { return !v026 && !hip && sm120_default(major, minor); }
+/// STRATA_PREFILL_LAYER_MAJOR (#3976: the layer-major prompt walk, which reads each layer's streamed experts once per super-chunk instead of once
+/// per chunk; another summation order of the routed experts, so KL-checked, not bitwise) when the user set nothing: ON for a CUDA layer split
+/// (`stages` = the number of GPUs the prompt path runs on, 2 or more), OFF on one GPU, in a HIP build and under STRATA_DEFAULTS=026. Measured on
+/// 2x RTX A4000 (sm_86) with Qwen3.8-Flash-Next IQ3_XXS and streamed experts only. The engine adds its own per-prompt declines (image prompts,
+/// DFlash2 taps, KVA, a batch read that yields, a peer GPU, no VRAM for a super-chunk): those read the chunk-major path as before.
+inline bool prefill_layer_major_default(bool hip, int stages, bool v026 = false) { return !v026 && !hip && stages >= 2; }
+
+/// The answer for STRATA_PREFILL_LAYER_MAJOR given the variable's text (nullptr = unset): any text that starts a number other than 0 is on, "0" (or any
+/// text that reads as 0) is off, both win over everything; unset or empty is the rule above. An explicit "1" is honoured on one GPU too (the
+/// opt-in of the first ports); a HIP build still declines at run time.
+inline bool prefill_layer_major_resolve(const char* env, bool hip, int stages, bool v026 = false) {
+ if (env != nullptr && env[0] != '\0') return std::atoi(env) != 0;
+ return prefill_layer_major_default(hip, stages, v026);
+}
+
} // namespace strata
diff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hpp
index da26dcc4c..549aff380 100644
--- a/include/strata/prefill/prefill.hpp
+++ b/include/strata/prefill/prefill.hpp
@@ -284,6 +284,7 @@ private:
bool run_layer_major(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err);
void lm_release();
bool lm_mid_ = false;
+ bool lm_by_default_ = false; ///< the walk is on because nothing was set (a wider VRAM margin, the measured super-chunk size)
// Stage-1 pipeline: intermediate stages return after handing their chunk to
// the direct successor. The public run() drains the chain once at prompt end.
bool run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::string& err);
diff --git a/src/core/arch_defaults_test.cpp b/src/core/arch_defaults_test.cpp
index 290f183d6..e13e76b1d 100644
--- a/src/core/arch_defaults_test.cpp
+++ b/src/core/arch_defaults_test.cpp
@@ -396,5 +396,30 @@ int main() {
CHECK(!prompt_attn_imma_default(false, 60)); // Pascal (P100)
CHECK(!prompt_attn_imma_default(true, 120)); // HIP
}
+ {
+ using namespace strata;
+ // the layer-major prompt walk (#3976): on by default for a CUDA layer split only
+ CHECK(prefill_layer_major_default(false, 2)); // 2x RTX A4000 / A6000: measured 5x time to first token
+ CHECK(prefill_layer_major_default(false, 4));
+ CHECK(!prefill_layer_major_default(false, 1)); // one GPU: opt-in only
+ CHECK(!prefill_layer_major_default(false, 0));
+ CHECK(!prefill_layer_major_default(true, 2)); // HIP builds
+ CHECK(!prefill_layer_major_default(false, 2, true)); // STRATA_DEFAULTS=026
+ CHECK(!prefill_layer_major_default(true, 1, true));
+ // the variable wins in both directions, whatever the rule says
+ CHECK(!prefill_layer_major_resolve("0", false, 2)); // the off switch
+ CHECK(prefill_layer_major_resolve("1", false, 1)); // the opt-in of the first ports: one GPU too
+ CHECK(prefill_layer_major_resolve("1", false, 2, true)); // explicit beats STRATA_DEFAULTS=026
+ CHECK(!prefill_layer_major_resolve("0", false, 2, false));
+ CHECK(!prefill_layer_major_resolve("0", true, 2));
+ CHECK(prefill_layer_major_resolve("2", false, 1)); // any non-zero number is on
+ CHECK(!prefill_layer_major_resolve("off", false, 2)); // text that reads as 0 is off (atoi), as before this rule existed
+ // unset or empty: the rule
+ CHECK(prefill_layer_major_resolve(nullptr, false, 2));
+ CHECK(prefill_layer_major_resolve("", false, 2));
+ CHECK(!prefill_layer_major_resolve(nullptr, false, 1));
+ CHECK(!prefill_layer_major_resolve(nullptr, false, 2, true));
+ CHECK(!prefill_layer_major_resolve(nullptr, true, 2));
+ }
return fails ? 1 : 0;
}
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index 2e424ff0e..16a609a5d 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -49,6 +49,8 @@
#include "strata/prefill/moe_mmq.hpp"
#include "strata/core/peer_experts.hpp"
#include "strata/core/expert_nibble.hpp"
+#include "strata/core/sm120_series.hpp"
+#include "strata/core/defaults_026.hpp"
#include "strata/prefill/kernels.hpp"
#include "strata/prefill/kva.hpp" // STRATA_KVA: the optional approximate-prefill plan (research, off by default)
@@ -5874,10 +5876,11 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
// other row groupings. on_chunk reports every sub-chunk in order from the final pass; while it reports one before
// the super-chunk's end, layer_major_mid() is true (the session state is the super-chunk's end: no checkpoint).
namespace {
-bool lm_env() {
- static const bool v = [] { const char* e = std::getenv("STRATA_PREFILL_LAYER_MAJOR"); return e && std::atoi(e) != 0; }();
- return v;
+const char* lm_env_text() { // STRATA_PREFILL_LAYER_MAJOR as set (nullptr: unset); the rule for unset is strata::prefill_layer_major_default
+ static const char* const e = std::getenv("STRATA_PREFILL_LAYER_MAJOR");
+ return e;
}
+bool lm_env_explicit() { const char* e = lm_env_text(); return e != nullptr && e[0] != '\0'; }
bool lm_even_env() { // even super-chunks (STRATA_PREFILL_LM_EVEN=0: the largest that fits, then the rest)
static const bool v = [] { const char* e = std::getenv("STRATA_PREFILL_LM_EVEN"); return !(e && e[0] == '0'); }();
@@ -5897,14 +5900,18 @@ int lm_r_env() { // 0 auto, 1 vram, 2 host
}();
return v;
}
-int64_t lm_margin_bytes() { // VRAM left free while a prompt runs (the verify windows, the driver)
- static const int64_t v = [] {
+// VRAM left free while a prompt runs (the verify windows, the driver): 96 MiB when STRATA_PREFILL_LAYER_MAJOR=1 asked for the walk, 512 MiB when
+// it is on by default (a user who set nothing gets the wider margin)
+int64_t lm_margin_bytes(bool by_default) {
+ static const int64_t explicit_v = [] {
const char* e = std::getenv("STRATA_PREFILL_LM_MARGIN_MIB");
- return (int64_t) (e ? std::atoll(e) : 96) << 20;
+ return e ? (int64_t) std::atoll(e) << 20 : (int64_t) -1;
}();
- return v;
+ return explicit_v >= 0 ? explicit_v : (int64_t) (by_default ? 512 : 96) << 20;
}
constexpr int64_t kLmDefaultTokens = 8192;
+// the super-chunk target when the walk is on by default and STRATA_PREFILL_LM_TOKENS is not set: the size measured (2x RTX A4000, 32K prompts)
+constexpr int64_t kLmDefaultOnTokens = 32768;
// the device tables of one layer, in int32s: row_tok (rows) | bounds (2 E) | u_tok (rows) | u_off (rows + E) |
// u_rows (rows) | u_w (rows), E = the most entries a layer can split into
int64_t lm_entries_max(int64_t rows, int64_t G) { return NE + rows / std::max<int64_t>(G, 1) + 2; }
@@ -5991,7 +5998,7 @@ int64_t Prefill::lm_prepare(int64_t n, std::string& why, int64_t cap) {
// (an explicit STRATA_PREFILL_LM_TOKENS may be one chunk: the A/B against the default path, experts aside)
const int64_t min_s = lm_tokens_env() > 0 ? m.T : 2 * m.T;
if (n < min_s) { why = "a short prompt"; return 0; }
- const int64_t want = lm_tokens_env() > 0 ? lm_tokens_env() : kLmDefaultTokens;
+ const int64_t want = lm_tokens_env() > 0 ? lm_tokens_env() : (lm_by_default_ ? kLmDefaultOnTokens : kLmDefaultTokens);
int64_t S = std::max<int64_t>(min_s, want / m.T * m.T);
S = std::min<int64_t>(S, (n + m.T - 1) / m.T * m.T);
if (cap > 0) {
@@ -6006,7 +6013,7 @@ int64_t Prefill::lm_prepare(int64_t n, std::string& why, int64_t cap) {
};
size_t free_b = 0, total_b = 0;
cudaMemGetInfo(&free_b, &total_b);
- const int64_t room = (int64_t) free_b - lm_margin_bytes();
+ const int64_t room = (int64_t) free_b - lm_margin_bytes(lm_by_default_);
m.lm.free_mib = (int64_t) (free_b >> 20);
auto fit = [&](bool r_dev) { // the largest super-chunk (a multiple of the chunk) that fits `room`
int64_t s = S;
@@ -7073,8 +7080,27 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
return ok;
}
#endif
- if (lm_env() && hand_in_ == nullptr && impl_) {
+#if defined(STRATA_USE_HIP) || defined(STRATA_HIP_GFX906)
+ constexpr bool lm_hip = true;
+#else
+ constexpr bool lm_hip = false;
+#endif
+ // layer-major prefill: STRATA_PREFILL_LAYER_MAJOR=1 / 0 win; unset, it is on for a CUDA layer split (sm120_series.hpp
+ // prefill_layer_major_default; STRATA_DEFAULTS=026 and a HIP build leave it off)
+ if (hand_in_ == nullptr && impl_ &&
+ strata::prefill_layer_major_resolve(lm_env_text(), lm_hip, stage_count(), strata::qwen_defaults_026())) {
+ const bool by_default = !lm_env_explicit();
+ for (Prefill* p = this; p != nullptr; p = p->next_) p->lm_by_default_ = by_default;
std::string why;
+ if (by_default) {
+ static bool noted = false;
+ if (!noted) {
+ noted = true;
+ std::fprintf(stderr, "strata prefill: layer-major prefill is on for this layer split (each layer's experts are read once per "
+ "super-chunk of up to %lld tokens, not once per chunk; STRATA_PREFILL_LAYER_MAJOR=0 or STRATA_DEFAULTS=026 "
+ "turns it off)\n", (long long) kLmDefaultOnTokens);
+ }
+ }
if (lm_prepare_chain(n, why) > 0) {
bool ok = run_layer_major(tokens, n, pos0, err);
std::string drain_err; // a layer split: the later stages' last super-chunk
@@ -7086,8 +7112,8 @@ bool Prefill::run(const int64_t* tokens, int64_t n, int64_t pos0, std::string& e
lm_release_chain();
return ok;
}
- static std::string said; // once per reason (a short prompt is the common one)
- if (why != said) {
+ static std::string said; // once per reason (a short prompt is the common one; not worth a line when nobody asked for the walk)
+ if (why != said && !(by_default && why == "a short prompt")) {
said = why;
std::fprintf(stderr, "strata prefill: layer-major off for this prompt: %s\n", why.c_str());
}