Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

spec: faster start-up adaptation of the expert tier in speculative runs (swap every 2 verify rounds for the first 200, then the normal interval); default on sm_86 only, STRATA_SPEC_ADAPT_EVERY / STRATA_SPEC_ADAPT_WARM set it elsewhere, an explicit --adapt-every is kept

Niko12212ccf57ec4682
diff --git a/include/strata/core/nv_defaults.hpp b/include/strata/core/nv_defaults.hpp
index d943f2875..4d0a591b5 100644
--- a/include/strata/core/nv_defaults.hpp
+++ b/include/strata/core/nv_defaults.hpp
@@ -28,3 +28,13 @@ std::vector<std::pair<std::string, std::string>> nv_default_env(int major, int m
std::vector<std::string> apply_nv_defaults(int major, int minor);
} // namespace strata::core
+
+namespace strata::core {
+
+/// dflash2-kern: the speculative runs' faster start-up adaptation of the expert tier (STRATA_SPEC_ADAPT_EVERY / _WARM, read by
+/// generate.cpp): sm_86 only (z440 A4000 Q2_0; the P100 is opt-in through the two variables, notes-dflash2-kern.md). Not part of
+/// the exact table above: the residency it changes decides which experts the GPU rounds, as the adaptive tier always does. The user's
+/// own settings and the cards' off switches (STRATA_SM86_PERF=0) is kept.
+void apply_spec_adapt_default(int major, int minor);
+
+} // namespace strata::core
diff --git a/src/core/nv_defaults.cpp b/src/core/nv_defaults.cpp
index 4b1a9f983..bc90f6623 100644
--- a/src/core/nv_defaults.cpp
+++ b/src/core/nv_defaults.cpp
@@ -70,4 +70,21 @@ std::vector<std::string> apply_nv_defaults(int major, int minor) {
return set;
}
+void apply_spec_adapt_default(int major, int minor) {
+ const int mj = strata::cc_major_of(major), mn = strata::cc_minor_of(minor);
+ // sm_86 only: the P100 (sm_60) gained 4-16% on JSON/code/chat/tool calls but lost 3% on the 4K agent transcript (5/5 pairs), so there it stays opt-in
+ const bool sm86 = mj == 8 && mn == 6;
+ if (!sm86) return;
+ if (off("STRATA_SM86_PERF")) return;
+ const std::pair<const char*, const char*> kv[2] = {{"STRATA_SPEC_ADAPT_EVERY", "2"}, {"STRATA_SPEC_ADAPT_WARM", "200"}};
+ for (const auto& p : kv) {
+ if (std::getenv(p.first) != nullptr) continue;
+#if defined(_WIN32)
+ _putenv_s(p.first, p.second);
+#else
+ setenv(p.first, p.second, 0);
+#endif
+ }
+}
+
} // namespace strata::core
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 19bfa2c53..edf3f52c8 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -684,6 +684,10 @@ struct Options {
/// Plan v0.3 P6: every `adapt_every` rounds, swap up to `adapt_swaps` of the most-routed missing experts into
/// the VRAM tier in place of the least-routed resident ones (decayed counts). 0 = static residency.
int adapt_every = 4;
+ bool adapt_every_given = false; ///< --adapt-every was on the command line (the per-card warm-up interval below does not override it)
+ int adapt_warm_every = 0; ///< dflash2-kern: > 0: the interval for the first adapt_warm_rounds verify rounds of a speculative run
+ int64_t adapt_warm_rounds = 0;
+ int adapt_iv(int64_t rounds) const { return adapt_warm_every > 0 && rounds < adapt_warm_rounds ? adapt_warm_every : adapt_every; } ///< the interval in force at verify round `rounds`
float adapt_decay = 0.7f; ///< the usage counts are multiplied by this after each adaptation (--adapt-decay)
/// an expert is swapped into the VRAM tier only when its decayed routing count beats the one it replaces by more than
/// this (--adapt-min-gain; both the blocking and the asynchronous tier)
@@ -2606,7 +2610,7 @@ int main(int argc, char** argv) {
else if (a == "--mtp") o.mtp = next("--mtp");
else if (a == "--mtp-window") o.mtp_window = std::atoll(next("--mtp-window"));
else if (a == "--pcie-frac") o.pcie_frac = std::atof(next("--pcie-frac"));
- else if (a == "--adapt-every") o.adapt_every = std::atoi(next("--adapt-every"));
+ else if (a == "--adapt-every") { o.adapt_every = std::atoi(next("--adapt-every")); o.adapt_every_given = true; }
else if (a == "--adapt-decay") o.adapt_decay = (float) std::atof(next("--adapt-decay"));
else if (a == "--adapt-min-gain") o.adapt_min_gain = (float) std::atof(next("--adapt-min-gain"));
else if (a == "--adapt-async") o.adapt_async = std::atoi(next("--adapt-async")) != 0 ? 1 : 0;
@@ -2810,7 +2814,7 @@ int main(int argc, char** argv) {
same = cudaDeviceGetAttribute(&m, cudaDevAttrComputeCapabilityMajor, d) == cudaSuccess &&
cudaDeviceGetAttribute(&n, cudaDevAttrComputeCapabilityMinor, d) == cudaSuccess && m == m0 && n == n0;
}
- if (same) strata::core::apply_nv_defaults(m0, n0);
+ if (same) { strata::core::apply_nv_defaults(m0, n0); strata::core::apply_spec_adapt_default(m0, n0); }
}
(void) cudaGetLastError();
}
@@ -3179,6 +3183,17 @@ int main(int argc, char** argv) {
const int chain_base = o.spec;
o.spec = std::max(chain_base, std::min(std::max(chain_base, o.mtp_max_t) + o.lookup_chain, 8)); // kVerifyMaxT
}
+ // dflash2-kern: right after the start the resident experts are the profile's, not this text's; a speculative round carries
+ // 2-5 tokens, so the adaptive tier (one swap round per adapt_every VERIFY rounds) learned the text 2-5 times slower per
+ // token. For the first STRATA_SPEC_ADAPT_WARM rounds it swaps every STRATA_SPEC_ADAPT_EVERY rounds (set per card by
+ // nv_defaults, where measured); later the normal interval. An explicit --adapt-every is kept as it is.
+ if (!o.adapt_every_given && o.spec >= 2 && o.adapt_every > 0) {
+ const char* e = std::getenv("STRATA_SPEC_ADAPT_EVERY");
+ const char* w = std::getenv("STRATA_SPEC_ADAPT_WARM");
+ const int n = e ? std::atoi(e) : 0;
+ const long long wr = w ? std::atoll(w) : 0;
+ if (n > 0 && n < o.adapt_every && wr > 0) { o.adapt_warm_every = n; o.adapt_warm_rounds = wr; }
+ }
strata::core::layer_set_shared_early(!o.shared_late);
if (!o.native_preset.empty()) {
try {
@@ -14292,7 +14307,7 @@ int main(int argc, char** argv) {
}
if (pl_adapt_thr.joinable()) return true; // still adapting
pl_apply();
- if (!drive.d.usage.empty() && o.adapt_every > 0 && (rounds % o.adapt_every) == 0 && pending.empty() &&
+ if (!drive.d.usage.empty() && o.adapt_every > 0 && (rounds % o.adapt_iv(rounds)) == 0 && pending.empty() &&
!exch_wait) {
pl_adapt_done = false;
pl_adapt_thr = std::thread([&] { strata::platform::name_this_thread("strata-adapt"); strata::aux_cpus::pin_current_thread(); pl_adapt_ok = adapt(); pl_adapt_done = true; });
@@ -14964,7 +14979,7 @@ int main(int argc, char** argv) {
std::thread adapt_thr; // the adaptive tier beside the commit and the draft (as in generate)
bool adapt_ok = true;
// (--adapt-async 1: the asynchronous tier above instead, ticked before the window)
- if (!drive.d.usage.empty() && !ajob && ((rounds + 1) % o.adapt_every) == 0)
+ if (!drive.d.usage.empty() && !ajob && ((rounds + 1) % o.adapt_iv(rounds)) == 0)
adapt_thr = std::thread([&] { strata::platform::name_this_thread("strata-adapt"); strata::aux_cpus::pin_current_thread(); adapt_ok = adapt(); });
if (!ver.commit(a + 1, err)) {
if (adapt_thr.joinable()) adapt_thr.join();
@@ -16393,7 +16408,7 @@ int main(int argc, char** argv) {
// GPU commits and drafts; it touches only the residency tables, which nothing reads until the next window
std::thread adapt_thr;
bool adapt_ok = true;
- if (!drive.d.usage.empty() && ((rounds + 1) % o.adapt_every) == 0)
+ if (!drive.d.usage.empty() && ((rounds + 1) % o.adapt_iv(rounds)) == 0)
adapt_thr = std::thread([&] { strata::platform::name_this_thread("strata-adapt"); strata::aux_cpus::pin_current_thread(); adapt_ok = adapt(); });
strata::core::progress_at("spec loop: commit", (int64_t) T);
if (!ver.commit(a + 1, err)) {