Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: faster mode (5 experts a word) is the default on a GPU that shares the PC's RAM (Strix Halo: decode +11..13%, prompts +5..7% against 6); the 16 GiB rule is unchanged, --experts N wins; rule test rows
Niko1221c4ce40bf5e23
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hppindex 33ca78772..06f003ac5 100644--- a/src/dsv41/ds4_experts_rule.hpp+++ b/src/dsv41/ds4_experts_rule.hpp@@ -2,7 +2,7 @@//// The model routes every word to its top 6 experts. On a PC with a small graphics card the CPU lane and the SSD set the speed, and// each expert dropped saves their bytes: the router's top N are kept and their weights renormalised (the same selection the full-// model makes, a shorter list). The default is 5 where the graphics memory is 16 GiB or less, the model's own count elsewhere;+// model makes, a shorter list). The default is 5 where the graphics memory is 16 GiB or less, or the GPU shares the PC's RAM, the model's own count elsewhere;// --experts N says it outright (--experts 6: full quality), DS4_TOPK=N is the test override over everything.#pragma once@@ -15,11 +15,12 @@ constexpr uint64_t kFasterModeVramMax = 16ull << 30; // GiB of graphics memoryconstexpr int kFasterModeExperts = 5;// requested: --experts (-1: not given, the rule decides); vram_bytes: the total memory of the cards the engine uses;-// model_k: the experts a word takes in the model. The result is in 1 .. model_k.-inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k) {+// model_k: the experts a word takes in the model; integrated: a GPU that shares the PC's RAM (Strix Halo: part of the experts live only on+// the SSD, each expert dropped is less read - measured 5 against 6 on a Radeon 8065S: decode +11..13%, prompts +5..7%). The result is in 1 .. model_k.+inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k, bool integrated = false) {if (model_k <= 1) return model_k;int n = requested;- if (n < 1) n = vram_bytes != 0 && vram_bytes <= kFasterModeVramMax ? kFasterModeExperts : model_k;+ if (n < 1) n = integrated || (vram_bytes != 0 && vram_bytes <= kFasterModeVramMax) ? kFasterModeExperts : model_k;if (n > model_k) n = model_k;return n < 1 ? 1 : n;}diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 053ab81f8..cc6fc9fbb 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -2046,8 +2046,13 @@ struct Model::Impl {if (!l.mtp) { // experts a word takes: --experts N / the faster-mode rule (ds4_experts_rule.hpp), DS4_TOPK=N over both (a test)if (il == 0) {uint64_t vram_all = 0;- for (const Dev& dv : devs) vram_all += (uint64_t) dv.total;- const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp), l.k_exp);+ bool integrated = false; // (a GPU that shares the PC's RAM: Strix Halo)+ for (const Dev& dv : devs) {+ vram_all += (uint64_t) dv.total;+ cudaDeviceProp ip{};+ if (cudaGetDeviceProperties(&ip, dv.id) == cudaSuccess && ip.integrated) integrated = true;+ }+ const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp, integrated), l.k_exp);experts_used = n;if (n < l.k_exp)std::fprintf(stderr, "ds4: experts per word: %d (%s; full quality: --experts %d)\n", n,diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cppindex dec47073f..d588c4a33 100644--- a/tests/dsv41/experts_rule_test.cpp+++ b/tests/dsv41/experts_rule_test.cpp@@ -30,6 +30,12 @@ int main() {CHECK_EQ(experts_per_word(-1, 24 * GiB, 6), 6);CHECK_EQ(experts_per_word(-1, 4 * 32 * GiB, 6), 6);CHECK_EQ(experts_per_word(-1, 2 * 12 * GiB, 6), 6); // two 12 GB cards hold 24 GiB in all+ // a GPU that shares the PC's RAM (Strix Halo, ~91 GiB visible): faster mode whatever the size; discrete cards by size only+ CHECK_EQ(experts_per_word(-1, 91 * GiB, 6, true), 5);+ CHECK_EQ(experts_per_word(-1, 24 * GiB, 6, false), 6); // a discrete 24 GB card+ CHECK_EQ(experts_per_word(-1, 12 * GiB, 6, false), 5); // a discrete 12 GB card+ CHECK_EQ(experts_per_word(6, 91 * GiB, 6, true), 6); // --experts 6 wins on the unified-memory GPU too+ CHECK_EQ(experts_per_word(-1, 91 * GiB, 4, true), 4); // never raised// unknown memory (0): the model's count, never a silent cutCHECK_EQ(experts_per_word(-1, 0, 6), 6);// the flag decides, both ways