Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
route: residency-biased routing is on by default at 2.0 on a card of 13 GiB or less
Niko1221c30656aaf908
diff --git a/src/core/verify.cpp b/src/core/verify.cppindex 3d6f48b2e..145e9d762 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -1966,9 +1966,23 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {mb.logits = logits_ + t * NE; mb.ids = ids_ + t * K; mb.weights = w_ + t * K;if (!moe_route(wt, g, l, K, mb, mixed_ + t * N, cs, err, nullptr)) return false;}- // STRATA_ROUTE_PRIOR=lambda (opt-in, changes the output): the top-10 picked again with a logit bonus for the- // resident experts; the weights stay the router's (route_prior.cu)- static const float route_prior = [] { const char* v = std::getenv("STRATA_ROUTE_PRIOR"); return v ? (float) std::atof(v) : 0.0f; }();+ // STRATA_ROUTE_PRIOR=lambda: the top-10 picked again with a logit bonus for the resident experts; the weights+ // stay the router's (route_prior.cu). **On by default at 2.0 on a card of 13 GiB or less**, where the expert+ // cache cannot hold the model: measured on an RTX 3060 12 GB (IQ3_XXS), 5 interleaved pairs, story 44.2 ->+ // 60.9 tok/s (+37.6%) and code 47.3 -> 64.0 (+35.2%); at 1.0 it is +22.6% / +24.6%. The owner approved the+ // quality trade for 12 GB cards (it changes which experts run). It is NOT a default where the cache holds+ // the model: on an RTX PRO 5000 72 GB it measured -1.9% / -0.5% and on 2x RTX A4000 -4.2% / +11.0%, so+ // STRATA_ROUTE_PRIOR=0 turns it off and any explicit value wins.+ static const float route_prior = [] {+ const char* v = std::getenv("STRATA_ROUTE_PRIOR");+ if (v != nullptr) return (float) std::atof(v);+ int dev = 0;+ if (cudaGetDevice(&dev) != cudaSuccess) return 0.0f;+ size_t free_b = 0, total_b = 0;+ if (cudaMemGetInfo(&free_b, &total_b) != cudaSuccess) return 0.0f;+ cudaGetLastError(); // drop what a query on a device without a context leaves behind+ return (double) total_b / 1073741824.0 <= 13.0 ? 2.0f : 0.0f;+ }();if (route_prior > 0.0f && NE == 512 && K == 10 && hits_.d_res != nullptr)route_prior_top10(logits_ + tb * NE, hits_.d_res + l * g.n_expert, route_prior, ids_ + tb * K, w_ + tb * K, n, cs);if (route_resident_cfg().margin > 0.0f && NE == 512 && K == 10 && hits_.d_res != nullptr) {