Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: --experts N (the router's top N, weights renormalised); default 5 where the cards hold 16 GiB or less in all, the model's 6 elsewhere; startup line 'experts per word: 5 (faster mode; full quality: --experts 6)'; DS4_TOPK=N stays as a test override; rule test; DS4_PROF prints the CPU lane's stage split

Niko12213dd11c6dc1f0
diff --git a/CMakeLists.txt b/CMakeLists.txt
index acc764463..1f180caa5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -888,6 +888,9 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
add_executable(ds4_slot_test tests/dsv41/slot_test.cpp)
target_include_directories(ds4_slot_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41)
target_compile_features(ds4_slot_test PRIVATE cxx_std_17)
+ add_executable(ds4_experts_rule_test tests/dsv41/experts_rule_test.cpp)
+ target_include_directories(ds4_experts_rule_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41)
+ target_compile_features(ds4_experts_rule_test PRIVATE cxx_std_17)
endif()
if(STRATA_ENABLE_HIP)
foreach(_t strata_dsv41_kernels strata-dsv41)
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hpp
new file mode 100644
index 000000000..316b6cdaa
--- /dev/null
+++ b/src/dsv41/ds4_experts_rule.hpp
@@ -0,0 +1,36 @@
+// Strata. Experts per word: the "faster mode" rule of the DeepSeek V4.1 engine. Header only, no GPU code (tests/dsv41/experts_rule_test.cpp).
+//
+// The model routes every word to its top 6 experts. On a PC with a small graphics card the CPU lane and the SSD set the speed, and
+// each expert dropped saves their bytes: the router's top N are kept and their weights renormalised (the same selection the full
+// model makes, a shorter list). The default is 5 where the graphics memory is 16 GiB or less, the model's own count elsewhere;
+// --experts N says it outright (--experts 6: full quality), DS4_TOPK=N is the test override over everything.
+#pragma once
+
+#include <cstdint>
+#include <cstdlib>
+
+namespace strata::ds4 {
+
+constexpr uint64_t kFasterModeVramMax = 16ull << 30; // GiB of graphics memory in all, at most
+constexpr int kFasterModeExperts = 5;
+
+// requested: --experts (-1: not given, the rule decides); vram_bytes: the total memory of the cards the engine uses;
+// model_k: the experts a word takes in the model. The result is in 1 .. model_k.
+inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k) {
+ if (model_k <= 1) return model_k;
+ int n = requested;
+ if (n < 1) n = vram_bytes != 0 && vram_bytes <= kFasterModeVramMax ? kFasterModeExperts : model_k;
+ if (n > model_k) n = model_k;
+ return n < 1 ? 1 : n;
+}
+
+// DS4_TOPK=N (a test, not for users): over the flag and the rule; returns the count to use
+inline int experts_test_override(int chosen, int model_k) {
+ if (const char* v = std::getenv("DS4_TOPK")) {
+ const int n = std::atoi(v);
+ if (n >= 1 && n <= model_k) return n;
+ }
+ return chosen;
+}
+
+} // namespace strata::ds4
diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cpp
index c253925e7..857df4128 100644
--- a/src/dsv41/ds4_main.cpp
+++ b/src/dsv41/ds4_main.cpp
@@ -86,7 +86,7 @@ void usage() {
std::fprintf(stderr,
"maya-ds4 --pack <model>/maya-ds4 [--gpus 0,1,...] [--max-context N] [--reserve-mib N]\n"
" [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--ssd-cache GB] [--ssd-policy lru|lfu|lfulru] [--no-expert-cache] [--verbose]\n"
- " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none|auto] [--dspark-block N] [--dspark-conf F]\n"
+ " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none|auto] [--dspark-block N] [--dspark-conf F] [--experts N|auto]\n"
" [--vision on|off] [--mmap-experts on|off|auto]\n"
" (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"
" | --encode-image FILE.rgb --size WxH --dump-image OUT\n"
@@ -1227,6 +1227,10 @@ int main(int argc, char** argv) {
o.dspark = v != "none"; // none: not even loaded
o.dspark_auto = v == "auto";
} else if (a == "--dspark-conf") o.dspark_conf = (float) std::atof(val("--dspark-conf").c_str());
+ else if (a == "--experts") {
+ const std::string v = val("--experts");
+ o.experts = v == "auto" ? -1 : std::max(1, std::atoi(v.c_str()));
+ }
else if (a == "--dspark-block") o.dspark_block = std::max(0, std::atoi(val("--dspark-block").c_str()));
else if (a == "--tp") o.tp = std::max(1, std::atoi(val("--tp").c_str()));
else if (a == "--prefill-chunk") o.prefill_chunk = std::max(0, std::atoi(val("--prefill-chunk").c_str()));
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index c77728257..2c3ec87b4 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -13,6 +13,7 @@
// h = post * moe + comb^T h; pre = f_pre
// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).
#include "ds4_model.hpp"
+#include "ds4_experts_rule.hpp"
#include "ds4_engram_io.hpp"
#include "ds4_gguf.cuh"
@@ -2010,6 +2011,7 @@ struct Model::Impl {
if (o.dspark_block > 0 && std::getenv("DS4_DSPARK_MAX") == nullptr) max_drafts = o.dspark_block; // (--dspark-block N: at most N drafts a round)
L.resize((size_t) (g.nl + g.n_mtp));
int last_src = -1;
+ int experts_used = 1 << 20;
for (int il = 0; il < g.nl + g.n_mtp; ++il) {
Layer& l = L[(size_t) il];
l.il = il;
@@ -2017,6 +2019,20 @@ struct Model::Impl {
l.name = l.mtp ? "mtp." + std::to_string(il - g.nl) + "." : "layers." + std::to_string(il) + ".";
l.n_exp = l.mtp ? g.d_n_exp : g.n_exp;
l.k_exp = l.mtp ? g.d_k : g.k_exp;
+ if (!l.mtp) { // experts a word takes: --experts N / the faster-mode rule (ds4_experts_rule.hpp), DS4_TOPK=N over both (a test)
+ if (il == 0) {
+ uint64_t vram_all = 0;
+ for (const Dev& dv : devs) vram_all += (uint64_t) dv.total;
+ const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp), l.k_exp);
+ experts_used = n;
+ if (n < l.k_exp)
+ std::fprintf(stderr, "ds4: experts per word: %d (%s; full quality: --experts %d)\n", n,
+ o.experts >= 1 ? "as asked" : "faster mode", l.k_exp);
+ else if (o.experts >= 1)
+ std::fprintf(stderr, "ds4: experts per word: %d (full quality)\n", n);
+ }
+ l.k_exp = std::min(l.k_exp, experts_used);
+ }
l.ratio = g.ratios[(size_t) il];
if (l.mtp) { // window attention only: no compressor, indexer or Engram
if (l.ratio != 0 || l.n_exp <= 0 || l.k_exp <= 0 || l.k_exp > kPicks) {
@@ -5531,6 +5547,7 @@ struct Model::Impl {
return;
}
const float lim = g.lim;
+ const double tp0 = now_ms();
{ // (os::prefetch_cold, as above: the resident ones are left alone - on Windows the hint walks every page of a
// range, ~0.13 ms an IQ2_XXS expert, on the lane's critical path before the first entry computes.
// off by default (the mmap path, owner PC: not faster in every pair); DS4_IQ_PF_COLD=1 turns it on)
@@ -5560,6 +5577,7 @@ struct Model::Impl {
gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));
}
}
+ const double tp1 = now_ms();
std::function<void(int)> f[4];
for (int X = 0; X < NH; ++X)
f[X] = [&, X](int j) {
@@ -5577,10 +5595,12 @@ struct Model::Impl {
r0 - X * FFh, r1 - X * FFh, lim);
};
run_nodes(ne * GB, f);
+ const double tp2 = now_ms();
for (const Ent& en : ents)
for (int i = 0; i < en.n; ++i)
gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,
aq + (size_t) (kWin + en.slot0 + i) * E * 4);
+ const double tp3 = now_ms();
for (int X = 0; X < NH; ++X)
f[X] = [&, X](int j) {
const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];
@@ -5594,6 +5614,16 @@ struct Model::Impl {
gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);
};
run_nodes(ne * DB, f);
+ { // DS4_PROF: where the CPU lane's time goes (act prep + prefetch, gate/up, h quant, down), summed
+ static const bool on = std::getenv("DS4_PROF") != nullptr;
+ if (on) {
+ static double acc[6] = {};
+ const double te = now_ms();
+ acc[0] += tp1 - tp0; acc[1] += tp2 - tp1; acc[2] += tp3 - tp2; acc[3] += te - tp3; acc[4] += 1; acc[5] += ne;
+ if (((long long) acc[4]) % 1000 == 0)
+ std::fprintf(stderr, "ds4 prof CPU lane (IQ rows) per call: n %.0f, entries/call %.2f, prep %.3f ms, gate+up %.3f ms, h-quant %.3f ms, down %.3f ms\n", acc[4], acc[5] / acc[4], acc[0] / acc[4], acc[1] / acc[4], acc[2] / acc[4], acc[3] / acc[4]);
+ }
+ }
}
// the window's CPU-lane experts made ready (and pinned) in the store for the length of the call
diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hpp
index 28fe665be..9cac562c2 100644
--- a/src/dsv41/ds4_model.hpp
+++ b/src/dsv41/ds4_model.hpp
@@ -42,6 +42,9 @@ struct Options {
// dspark_auto: the drafter is loaded only where the routed experts fit the GPUs (more than 1.5x what they have, the verify rounds
// read 3.3x the bytes of a token and measured no faster than plain decoding, and the drafter's 7 GB of experts take the VRAM the
// trunk's hottest would have had); --dspark on / off / none says it outright
+ // experts a word takes: -1 the rule (5 where the cards hold 16 GiB or less in all, else the model's 6), N the router's top N with
+ // their weights renormalised (--experts 6: full quality); ds4_experts_rule.hpp
+ int experts = -1;
bool dspark_auto = true;
bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decoding
int dspark_block = 0; // at most this many drafts a round (--dspark-block N; 0: the pack's block size)
diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cpp
new file mode 100644
index 000000000..f1fdc6d34
--- /dev/null
+++ b/tests/dsv41/experts_rule_test.cpp
@@ -0,0 +1,68 @@
+// Strata. The experts-per-word rule of the DeepSeek V4.1 engine (src/dsv41/ds4_experts_rule.hpp): 5 on a card with 16 GiB or less, 6
+// elsewhere, --experts overrides, DS4_TOPK over both. No GPU needed.
+#include "ds4_experts_rule.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+
+using strata::ds4::experts_per_word;
+using strata::ds4::experts_test_override;
+
+static int fails = 0;
+#define CHECK_EQ(a, b) \
+ do { \
+ const long long x = (a), y = (b); \
+ if (x != y) { \
+ std::printf("FAIL %s:%d: %s = %lld, expected %lld\n", __FILE__, __LINE__, #a, x, y); \
+ ++fails; \
+ } \
+ } while (0)
+
+int main() {
+ const uint64_t GiB = 1ull << 30;
+ // the owner's 12 GB-class cards (RTX 5070 12227 MiB, RTX 3060 12 GB), 16 GiB cards: faster mode by default
+ CHECK_EQ(experts_per_word(-1, 12227ull << 20, 6), 5);
+ CHECK_EQ(experts_per_word(-1, 12 * GiB, 6), 5);
+ CHECK_EQ(experts_per_word(-1, 16 * GiB, 6), 5);
+ CHECK_EQ(experts_per_word(-1, 8 * GiB, 6), 5);
+ // 24 GB and bigger, and several cards: the model's own 6
+ CHECK_EQ(experts_per_word(-1, 16 * GiB + 1, 6), 6);
+ CHECK_EQ(experts_per_word(-1, 24 * GiB, 6), 6);
+ CHECK_EQ(experts_per_word(-1, 4 * 32 * GiB, 6), 6);
+ CHECK_EQ(experts_per_word(-1, 2 * 12 * GiB, 6), 6); // two 12 GB cards hold 24 GiB in all
+ // unknown memory (0): the model's count, never a silent cut
+ CHECK_EQ(experts_per_word(-1, 0, 6), 6);
+ // the flag decides, both ways
+ CHECK_EQ(experts_per_word(6, 12 * GiB, 6), 6);
+ CHECK_EQ(experts_per_word(4, 12 * GiB, 6), 4);
+ CHECK_EQ(experts_per_word(5, 96 * GiB, 6), 5);
+ CHECK_EQ(experts_per_word(9, 12 * GiB, 6), 6); // never more than the model has
+ CHECK_EQ(experts_per_word(1, 12 * GiB, 6), 1);
+ // a model that routes to 1 (or no) expert: left alone
+ CHECK_EQ(experts_per_word(-1, 12 * GiB, 1), 1);
+ CHECK_EQ(experts_per_word(-1, 12 * GiB, 0), 0);
+ // a model that routes to fewer than 5: the rule never raises it
+ CHECK_EQ(experts_per_word(-1, 12 * GiB, 4), 4);
+ // the test override wins over the rule and the flag
+#if defined(_WIN32)
+ _putenv("DS4_TOPK=4");
+#else
+ setenv("DS4_TOPK", "4", 1);
+#endif
+ CHECK_EQ(experts_test_override(experts_per_word(-1, 96 * GiB, 6), 6), 4);
+ CHECK_EQ(experts_test_override(experts_per_word(6, 12 * GiB, 6), 6), 4);
+#if defined(_WIN32)
+ _putenv("DS4_TOPK=9");
+#else
+ setenv("DS4_TOPK", "9", 1);
+#endif
+ CHECK_EQ(experts_test_override(5, 6), 5); // out of range: ignored
+#if defined(_WIN32)
+ _putenv("DS4_TOPK=");
+#else
+ unsetenv("DS4_TOPK");
+#endif
+ CHECK_EQ(experts_test_override(5, 6), 5);
+ if (fails == 0) std::printf("experts rule: all checks passed\n");
+ return fails == 0 ? 0 : 1;
+}