Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

route: residency-biased routing is on by default at 2.0 on a card of 13 GiB or less

Niko1221c30656aaf908
diff --git a/src/core/verify.cpp b/src/core/verify.cpp
index 3d6f48b2e..145e9d762 100644
--- a/src/core/verify.cpp
+++ b/src/core/verify.cpp
@@ -1966,9 +1966,23 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
mb.logits = logits_ + t * NE; mb.ids = ids_ + t * K; mb.weights = w_ + t * K;
if (!moe_route(wt, g, l, K, mb, mixed_ + t * N, cs, err, nullptr)) return false;
}
- // STRATA_ROUTE_PRIOR=lambda (opt-in, changes the output): the top-10 picked again with a logit bonus for the
- // resident experts; the weights stay the router's (route_prior.cu)
- static const float route_prior = [] { const char* v = std::getenv("STRATA_ROUTE_PRIOR"); return v ? (float) std::atof(v) : 0.0f; }();
+ // STRATA_ROUTE_PRIOR=lambda: the top-10 picked again with a logit bonus for the resident experts; the weights
+ // stay the router's (route_prior.cu). **On by default at 2.0 on a card of 13 GiB or less**, where the expert
+ // cache cannot hold the model: measured on an RTX 3060 12 GB (IQ3_XXS), 5 interleaved pairs, story 44.2 ->
+ // 60.9 tok/s (+37.6%) and code 47.3 -> 64.0 (+35.2%); at 1.0 it is +22.6% / +24.6%. The owner approved the
+ // quality trade for 12 GB cards (it changes which experts run). It is NOT a default where the cache holds
+ // the model: on an RTX PRO 5000 72 GB it measured -1.9% / -0.5% and on 2x RTX A4000 -4.2% / +11.0%, so
+ // STRATA_ROUTE_PRIOR=0 turns it off and any explicit value wins.
+ static const float route_prior = [] {
+ const char* v = std::getenv("STRATA_ROUTE_PRIOR");
+ if (v != nullptr) return (float) std::atof(v);
+ int dev = 0;
+ if (cudaGetDevice(&dev) != cudaSuccess) return 0.0f;
+ size_t free_b = 0, total_b = 0;
+ if (cudaMemGetInfo(&free_b, &total_b) != cudaSuccess) return 0.0f;
+ cudaGetLastError(); // drop what a query on a device without a context leaves behind
+ return (double) total_b / 1073741824.0 <= 13.0 ? 2.0f : 0.0f;
+ }();
if (route_prior > 0.0f && NE == 512 && K == 10 && hits_.d_res != nullptr)
route_prior_top10(logits_ + tb * NE, hits_.d_res + l * g.n_expert, route_prior, ids_ + tb * K, w_ + tb * K, n, cs);
if (route_resident_cfg().margin > 0.0f && NE == 512 && K == 10 && hits_.d_res != nullptr) {