Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: faster mode (5 experts a word) is the default on a GPU that shares the PC's RAM (Strix Halo: decode +11..13%, prompts +5..7% against 6); the 16 GiB rule is unchanged, --experts N wins; rule test rows

Niko1221c4ce40bf5e23
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hpp
index 33ca78772..06f003ac5 100644
--- a/src/dsv41/ds4_experts_rule.hpp
+++ b/src/dsv41/ds4_experts_rule.hpp
@@ -2,7 +2,7 @@
//
// The model routes every word to its top 6 experts. On a PC with a small graphics card the CPU lane and the SSD set the speed, and
// each expert dropped saves their bytes: the router's top N are kept and their weights renormalised (the same selection the full
-// model makes, a shorter list). The default is 5 where the graphics memory is 16 GiB or less, the model's own count elsewhere;
+// model makes, a shorter list). The default is 5 where the graphics memory is 16 GiB or less, or the GPU shares the PC's RAM, the model's own count elsewhere;
// --experts N says it outright (--experts 6: full quality), DS4_TOPK=N is the test override over everything.
#pragma once
@@ -15,11 +15,12 @@ constexpr uint64_t kFasterModeVramMax = 16ull << 30; // GiB of graphics memory
constexpr int kFasterModeExperts = 5;
// requested: --experts (-1: not given, the rule decides); vram_bytes: the total memory of the cards the engine uses;
-// model_k: the experts a word takes in the model. The result is in 1 .. model_k.
-inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k) {
+// model_k: the experts a word takes in the model; integrated: a GPU that shares the PC's RAM (Strix Halo: part of the experts live only on
+// the SSD, each expert dropped is less read - measured 5 against 6 on a Radeon 8065S: decode +11..13%, prompts +5..7%). The result is in 1 .. model_k.
+inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k, bool integrated = false) {
if (model_k <= 1) return model_k;
int n = requested;
- if (n < 1) n = vram_bytes != 0 && vram_bytes <= kFasterModeVramMax ? kFasterModeExperts : model_k;
+ if (n < 1) n = integrated || (vram_bytes != 0 && vram_bytes <= kFasterModeVramMax) ? kFasterModeExperts : model_k;
if (n > model_k) n = model_k;
return n < 1 ? 1 : n;
}
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 053ab81f8..cc6fc9fbb 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -2046,8 +2046,13 @@ struct Model::Impl {
if (!l.mtp) { // experts a word takes: --experts N / the faster-mode rule (ds4_experts_rule.hpp), DS4_TOPK=N over both (a test)
if (il == 0) {
uint64_t vram_all = 0;
- for (const Dev& dv : devs) vram_all += (uint64_t) dv.total;
- const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp), l.k_exp);
+ bool integrated = false; // (a GPU that shares the PC's RAM: Strix Halo)
+ for (const Dev& dv : devs) {
+ vram_all += (uint64_t) dv.total;
+ cudaDeviceProp ip{};
+ if (cudaGetDeviceProperties(&ip, dv.id) == cudaSuccess && ip.integrated) integrated = true;
+ }
+ const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp, integrated), l.k_exp);
experts_used = n;
if (n < l.k_exp)
std::fprintf(stderr, "ds4: experts per word: %d (%s; full quality: --experts %d)\n", n,
diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cpp
index dec47073f..d588c4a33 100644
--- a/tests/dsv41/experts_rule_test.cpp
+++ b/tests/dsv41/experts_rule_test.cpp
@@ -30,6 +30,12 @@ int main() {
CHECK_EQ(experts_per_word(-1, 24 * GiB, 6), 6);
CHECK_EQ(experts_per_word(-1, 4 * 32 * GiB, 6), 6);
CHECK_EQ(experts_per_word(-1, 2 * 12 * GiB, 6), 6); // two 12 GB cards hold 24 GiB in all
+ // a GPU that shares the PC's RAM (Strix Halo, ~91 GiB visible): faster mode whatever the size; discrete cards by size only
+ CHECK_EQ(experts_per_word(-1, 91 * GiB, 6, true), 5);
+ CHECK_EQ(experts_per_word(-1, 24 * GiB, 6, false), 6); // a discrete 24 GB card
+ CHECK_EQ(experts_per_word(-1, 12 * GiB, 6, false), 5); // a discrete 12 GB card
+ CHECK_EQ(experts_per_word(6, 91 * GiB, 6, true), 6); // --experts 6 wins on the unified-memory GPU too
+ CHECK_EQ(experts_per_word(-1, 91 * GiB, 4, true), 4); // never raised
// unknown memory (0): the model's count, never a silent cut
CHECK_EQ(experts_per_word(-1, 0, 6), 6);
// the flag decides, both ways