Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 CPU lane: two cores left free beside the host thread when the lane would take every logical CPU; STRATA_DS4_LANE_FREE=0 as before
Niko1221e51e84e22a7d
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex b73c1ca4e..fb498cead 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -4560,11 +4560,25 @@ struct Model::Impl {cs.assign((size_t) NH, {});for (size_t i = 0; i < all.size(); ++i) cs[i * (size_t) NH / all.size()].push_back(all[i]);}+ static const int lane_free = [] {+ const char* v = std::getenv("STRATA_DS4_LANE_FREE");+ return v != nullptr ? std::max(0, std::atoi(v)) : 2;+ }();+ size_t lane_cpus = 0;+ for (const auto& v : cs) lane_cpus += v.size();+ const bool lane_takes_all = lane_cpus >= (size_t) std::max(1u, std::thread::hardware_concurrency());for (int X = 0; X < (int) cs.size() && X < std::max(NH, 1); ++X) {std::vector<int> c = cs[(size_t) X];if (file_experts && NH == 1)for (size_t Y = 1; Y < cs.size(); ++Y) c.insert(c.end(), cs[Y].begin(), cs[Y].end());if (X == 0 && !c.empty()) c.erase(c.begin()); // the host thread's core (it helps node 0's pool)+ // cores of the first pool left free beside the host thread's, for the thread that launches the decode's kernels+ // (and waits on the device) and whatever else the PC runs: with a worker spinning on every other logical CPU+ // that thread waits for a core (Project Maya #359, a 20-core Windows 11 PC: decode 28.3 -> 55.5 tok/s). Only+ // when the lane would take every logical CPU and --cpu-threads is not given.+ // STRATA_DS4_LANE_FREE: the cores left free (2 by default, 0: as before)+ if (X == 0 && o.cpu_threads <= 0 && lane_free > 0 && (int) c.size() > lane_free + 1 && lane_takes_all)+ c.resize(c.size() - (size_t) lane_free);int nw = (int) c.size();if (o.cpu_threads > 0) nw = std::max(X == 0 ? 0 : 1, std::min(nw, o.cpu_threads / NH - (X == 0 ? 1 : 0)));pools.push_back(std::make_unique<Pool>(nw, c, 20000)); // spin through a token's layers (20 ms)