Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: the copy-all fit check is one tested rule (copy_all_fits) and also needs the Windows commit room (RAM + page file beside its margin), not only the free RAM, so a 64 GB PC with a 12 GB card, or a big-RAM PC with a small page file, never copies the lane in by itself; the auto store says (verbose) when it holds less than the CPU lane
Niko1221025cf7d95455
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hppindex 944468f1f..60d5881c3 100644--- a/src/dsv41/ds4_experts_rule.hpp+++ b/src/dsv41/ds4_experts_rule.hpp@@ -50,6 +50,15 @@ inline int prefill_chunk_default(bool own_default, double gpu_share, bool gguf_creturn file_experts ? big : 2048;}+// --mmap-experts auto copies all the CPU lane's experts into RAM at load (no demand store) only when they fit: lane_bytes of experts,+// ram_avail the RAM this process can still get (~0: unknown -> no), ram_margin left to the rest, commit_room what Windows still lets it+// commit beside its margin (1e300: no such limit, Linux). All or none: a lane that does not fit both leaves the demand store as it was.+// Measured 1 x R9700 + 251 GB, 2.00 pack: decode +10-25% (the copies sit on huge pages, the store's page-locked arena does not).+inline bool copy_all_fits(double lane_bytes, uint64_t ram_avail, double ram_margin, double commit_room) {+ if (!(lane_bytes > 0.0) || ram_avail == ~uint64_t{0}) return false;+ return lane_bytes + ram_margin <= (double) ram_avail && lane_bytes <= commit_room;+}+// DS4_TOPK=N (a test, not for users): over the flag and the rule; returns the count to useinline int experts_test_override(int chosen, int model_k) {if (const char* v = std::getenv("DS4_TOPK")) {diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex f1fcfd60b..b8d3991d8 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -4584,7 +4584,7 @@ struct Model::Impl {const double share = gpu_expert_share(&exp_b, &room, &other_b);if (exp_b <= 0) return false;const double lane = exp_b * (1.0 - std::min(1.0, share));- return lane > 0 && lane + (double) kRamMargin <= (double) avail;+ return copy_all_fits(lane, avail, (double) kRamMargin, commit_room()); // (Windows: the commit room too, not only the free RAM)}mutable bool auto_copy_all = false; // (set by store_gb(): the auto size is the CPU lane's experts copied in, no demand store)double store_cap_gb() const {diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cppindex b18c46f27..f0fad2dc5 100644--- a/tests/dsv41/experts_rule_test.cpp+++ b/tests/dsv41/experts_rule_test.cpp@@ -7,6 +7,7 @@using strata::ds4::experts_per_word;using strata::ds4::experts_test_override;+using strata::ds4::copy_all_fits;static int fails = 0;#define CHECK_EQ(a, b) \@@ -107,6 +108,24 @@ int main() {CHECK_EQ(prefill_chunk_default(false, 0.30, false, false, false, 24 * GiB), 2048);// the rule takes no --max-context: a 128K session on a 12 GB card gets the same chunk, because the chunk's buffers and the KV are// allocated whole at load for that context and a chunk that does not fit there is halved before the first prompt (4096 -> 2048 -> 1024 ...)+ // copy_all_fits: the 251 GB R9700 box (108 GB lane) copies; a 64 GB PC, a 125 GB box with the 2.25 pack's lane, an unknown RAM do not+ {+ const double GB = 1e9, margin = 8.0 * (1ull << 30);+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (236 * GB), margin, 1e300), 1);+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (110 * GB), margin, 1e300), 0); // inside the margin+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (116 * GB), margin, 1e300), 0); // 108 + 8.6 > 116+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (118 * GB), margin, 1e300), 1);+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (101 * GB), margin, 1e300), 0); // the 125 GB box, free RAM 101 GB+ CHECK_EQ(copy_all_fits(92 * GB, ~uint64_t{0}, margin, 1e300), 0); // RAM not known+ CHECK_EQ(copy_all_fits(0, (uint64_t) (236 * GB), margin, 1e300), 0); // nothing on the CPU lane+ // Windows: the commit room (RAM + page file beside its margin) must hold the lane too, whatever the free RAM says+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (236 * GB), margin, 90 * GB), 0);+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (236 * GB), margin, 108 * GB), 1);+ // the owner-PC class: 64 GB RAM, a 12 GB card (a lane of ~120 GB), 40 GB free, a 90 GB commit limit+ CHECK_EQ(copy_all_fits(120 * GB, (uint64_t) (40 * GB), margin, 60 * GB), 0);+ // a 192 GB Windows PC with a 4 GB page file: 150 GB free RAM but only 100 GB of commit room: no+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (150 * GB), margin, 100 * GB), 0);+ }if (fails == 0) std::printf("experts rule: all checks passed\n");return fails == 0 ? 0 : 1;}