Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: the copy-all fit check is one tested rule (copy_all_fits) and also needs the Windows commit room (RAM + page file beside its margin), not only the free RAM, so a 64 GB PC with a 12 GB card, or a big-RAM PC with a small page file, never copies the lane in by itself; the auto store says (verbose) when it holds less than the CPU lane

Niko1221025cf7d95455
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hpp
index 944468f1f..60d5881c3 100644
--- a/src/dsv41/ds4_experts_rule.hpp
+++ b/src/dsv41/ds4_experts_rule.hpp
@@ -50,6 +50,15 @@ inline int prefill_chunk_default(bool own_default, double gpu_share, bool gguf_c
return file_experts ? big : 2048;
}
+// --mmap-experts auto copies all the CPU lane's experts into RAM at load (no demand store) only when they fit: lane_bytes of experts,
+// ram_avail the RAM this process can still get (~0: unknown -> no), ram_margin left to the rest, commit_room what Windows still lets it
+// commit beside its margin (1e300: no such limit, Linux). All or none: a lane that does not fit both leaves the demand store as it was.
+// Measured 1 x R9700 + 251 GB, 2.00 pack: decode +10-25% (the copies sit on huge pages, the store's page-locked arena does not).
+inline bool copy_all_fits(double lane_bytes, uint64_t ram_avail, double ram_margin, double commit_room) {
+ if (!(lane_bytes > 0.0) || ram_avail == ~uint64_t{0}) return false;
+ return lane_bytes + ram_margin <= (double) ram_avail && lane_bytes <= commit_room;
+}
+
// DS4_TOPK=N (a test, not for users): over the flag and the rule; returns the count to use
inline int experts_test_override(int chosen, int model_k) {
if (const char* v = std::getenv("DS4_TOPK")) {
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index f1fcfd60b..b8d3991d8 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -4584,7 +4584,7 @@ struct Model::Impl {
const double share = gpu_expert_share(&exp_b, &room, &other_b);
if (exp_b <= 0) return false;
const double lane = exp_b * (1.0 - std::min(1.0, share));
- return lane > 0 && lane + (double) kRamMargin <= (double) avail;
+ return copy_all_fits(lane, avail, (double) kRamMargin, commit_room()); // (Windows: the commit room too, not only the free RAM)
}
mutable bool auto_copy_all = false; // (set by store_gb(): the auto size is the CPU lane's experts copied in, no demand store)
double store_cap_gb() const {
diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cpp
index b18c46f27..f0fad2dc5 100644
--- a/tests/dsv41/experts_rule_test.cpp
+++ b/tests/dsv41/experts_rule_test.cpp
@@ -7,6 +7,7 @@
using strata::ds4::experts_per_word;
using strata::ds4::experts_test_override;
+using strata::ds4::copy_all_fits;
static int fails = 0;
#define CHECK_EQ(a, b) \
@@ -107,6 +108,24 @@ int main() {
CHECK_EQ(prefill_chunk_default(false, 0.30, false, false, false, 24 * GiB), 2048);
// the rule takes no --max-context: a 128K session on a 12 GB card gets the same chunk, because the chunk's buffers and the KV are
// allocated whole at load for that context and a chunk that does not fit there is halved before the first prompt (4096 -> 2048 -> 1024 ...)
+ // copy_all_fits: the 251 GB R9700 box (108 GB lane) copies; a 64 GB PC, a 125 GB box with the 2.25 pack's lane, an unknown RAM do not
+ {
+ const double GB = 1e9, margin = 8.0 * (1ull << 30);
+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (236 * GB), margin, 1e300), 1);
+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (110 * GB), margin, 1e300), 0); // inside the margin
+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (116 * GB), margin, 1e300), 0); // 108 + 8.6 > 116
+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (118 * GB), margin, 1e300), 1);
+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (101 * GB), margin, 1e300), 0); // the 125 GB box, free RAM 101 GB
+ CHECK_EQ(copy_all_fits(92 * GB, ~uint64_t{0}, margin, 1e300), 0); // RAM not known
+ CHECK_EQ(copy_all_fits(0, (uint64_t) (236 * GB), margin, 1e300), 0); // nothing on the CPU lane
+ // Windows: the commit room (RAM + page file beside its margin) must hold the lane too, whatever the free RAM says
+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (236 * GB), margin, 90 * GB), 0);
+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (236 * GB), margin, 108 * GB), 1);
+ // the owner-PC class: 64 GB RAM, a 12 GB card (a lane of ~120 GB), 40 GB free, a 90 GB commit limit
+ CHECK_EQ(copy_all_fits(120 * GB, (uint64_t) (40 * GB), margin, 60 * GB), 0);
+ // a 192 GB Windows PC with a 4 GB page file: 150 GB free RAM but only 100 GB of commit room: no
+ CHECK_EQ(copy_all_fits(108 * GB, (uint64_t) (150 * GB), margin, 100 * GB), 0);
+ }
if (fails == 0) std::printf("experts rule: all checks passed\n");
return fails == 0 ? 0 : 1;
}