Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: --experts N (the router's top N, weights renormalised); default 5 where the cards hold 16 GiB or less in all, the model's 6 elsewhere; startup line 'experts per word: 5 (faster mode; full quality: --experts 6)'; DS4_TOPK=N stays as a test override; rule test; DS4_PROF prints the CPU lane's stage split
Niko12213dd11c6dc1f0
diff --git a/CMakeLists.txt b/CMakeLists.txtindex acc764463..1f180caa5 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -888,6 +888,9 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)add_executable(ds4_slot_test tests/dsv41/slot_test.cpp)target_include_directories(ds4_slot_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41)target_compile_features(ds4_slot_test PRIVATE cxx_std_17)+ add_executable(ds4_experts_rule_test tests/dsv41/experts_rule_test.cpp)+ target_include_directories(ds4_experts_rule_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/dsv41)+ target_compile_features(ds4_experts_rule_test PRIVATE cxx_std_17)endif()if(STRATA_ENABLE_HIP)foreach(_t strata_dsv41_kernels strata-dsv41)diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hppnew file mode 100644index 000000000..316b6cdaa--- /dev/null+++ b/src/dsv41/ds4_experts_rule.hpp@@ -0,0 +1,36 @@+// Strata. Experts per word: the "faster mode" rule of the DeepSeek V4.1 engine. Header only, no GPU code (tests/dsv41/experts_rule_test.cpp).+//+// The model routes every word to its top 6 experts. On a PC with a small graphics card the CPU lane and the SSD set the speed, and+// each expert dropped saves their bytes: the router's top N are kept and their weights renormalised (the same selection the full+// model makes, a shorter list). The default is 5 where the graphics memory is 16 GiB or less, the model's own count elsewhere;+// --experts N says it outright (--experts 6: full quality), DS4_TOPK=N is the test override over everything.+#pragma once++#include <cstdint>+#include <cstdlib>++namespace strata::ds4 {++constexpr uint64_t kFasterModeVramMax = 16ull << 30; // GiB of graphics memory in all, at most+constexpr int kFasterModeExperts = 5;++// requested: --experts (-1: not given, the rule decides); vram_bytes: the total memory of the cards the engine uses;+// model_k: the experts a word takes in the model. The result is in 1 .. model_k.+inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k) {+ if (model_k <= 1) return model_k;+ int n = requested;+ if (n < 1) n = vram_bytes != 0 && vram_bytes <= kFasterModeVramMax ? kFasterModeExperts : model_k;+ if (n > model_k) n = model_k;+ return n < 1 ? 1 : n;+}++// DS4_TOPK=N (a test, not for users): over the flag and the rule; returns the count to use+inline int experts_test_override(int chosen, int model_k) {+ if (const char* v = std::getenv("DS4_TOPK")) {+ const int n = std::atoi(v);+ if (n >= 1 && n <= model_k) return n;+ }+ return chosen;+}++} // namespace strata::ds4diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex c253925e7..857df4128 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -86,7 +86,7 @@ void usage() {std::fprintf(stderr,"maya-ds4 --pack <model>/maya-ds4 [--gpus 0,1,...] [--max-context N] [--reserve-mib N]\n"" [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--ssd-cache GB] [--ssd-policy lru|lfu|lfulru] [--no-expert-cache] [--verbose]\n"- " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none|auto] [--dspark-block N] [--dspark-conf F]\n"+ " [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none|auto] [--dspark-block N] [--dspark-conf F] [--experts N|auto]\n"" [--vision on|off] [--mmap-experts on|off|auto]\n"" (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"" | --encode-image FILE.rgb --size WxH --dump-image OUT\n"@@ -1227,6 +1227,10 @@ int main(int argc, char** argv) {o.dspark = v != "none"; // none: not even loadedo.dspark_auto = v == "auto";} else if (a == "--dspark-conf") o.dspark_conf = (float) std::atof(val("--dspark-conf").c_str());+ else if (a == "--experts") {+ const std::string v = val("--experts");+ o.experts = v == "auto" ? -1 : std::max(1, std::atoi(v.c_str()));+ }else if (a == "--dspark-block") o.dspark_block = std::max(0, std::atoi(val("--dspark-block").c_str()));else if (a == "--tp") o.tp = std::max(1, std::atoi(val("--tp").c_str()));else if (a == "--prefill-chunk") o.prefill_chunk = std::max(0, std::atoi(val("--prefill-chunk").c_str()));diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex c77728257..2c3ec87b4 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -13,6 +13,7 @@// h = post * moe + comb^T h; pre = f_pre// and after the last layer logits = head(rmsnorm(sum_s pre[s] h[s])).#include "ds4_model.hpp"+#include "ds4_experts_rule.hpp"#include "ds4_engram_io.hpp"#include "ds4_gguf.cuh"@@ -2010,6 +2011,7 @@ struct Model::Impl {if (o.dspark_block > 0 && std::getenv("DS4_DSPARK_MAX") == nullptr) max_drafts = o.dspark_block; // (--dspark-block N: at most N drafts a round)L.resize((size_t) (g.nl + g.n_mtp));int last_src = -1;+ int experts_used = 1 << 20;for (int il = 0; il < g.nl + g.n_mtp; ++il) {Layer& l = L[(size_t) il];l.il = il;@@ -2017,6 +2019,20 @@ struct Model::Impl {l.name = l.mtp ? "mtp." + std::to_string(il - g.nl) + "." : "layers." + std::to_string(il) + ".";l.n_exp = l.mtp ? g.d_n_exp : g.n_exp;l.k_exp = l.mtp ? g.d_k : g.k_exp;+ if (!l.mtp) { // experts a word takes: --experts N / the faster-mode rule (ds4_experts_rule.hpp), DS4_TOPK=N over both (a test)+ if (il == 0) {+ uint64_t vram_all = 0;+ for (const Dev& dv : devs) vram_all += (uint64_t) dv.total;+ const int n = experts_test_override(experts_per_word(o.experts, vram_all, l.k_exp), l.k_exp);+ experts_used = n;+ if (n < l.k_exp)+ std::fprintf(stderr, "ds4: experts per word: %d (%s; full quality: --experts %d)\n", n,+ o.experts >= 1 ? "as asked" : "faster mode", l.k_exp);+ else if (o.experts >= 1)+ std::fprintf(stderr, "ds4: experts per word: %d (full quality)\n", n);+ }+ l.k_exp = std::min(l.k_exp, experts_used);+ }l.ratio = g.ratios[(size_t) il];if (l.mtp) { // window attention only: no compressor, indexer or Engramif (l.ratio != 0 || l.n_exp <= 0 || l.k_exp <= 0 || l.k_exp > kPicks) {@@ -5531,6 +5547,7 @@ struct Model::Impl {return;}const float lim = g.lim;+ const double tp0 = now_ms();{ // (os::prefetch_cold, as above: the resident ones are left alone - on Windows the hint walks every page of a// range, ~0.13 ms an IQ2_XXS expert, on the lane's critical path before the first entry computes.// off by default (the mmap path, owner PC: not faster in every pair); DS4_IQ_PF_COLD=1 turns it on)@@ -5560,6 +5577,7 @@ struct Model::Impl {gg::iq_cpu_quant(fm.gu_act, x + (size_t) t * ldx, E, x_act(en, i));}}+ const double tp1 = now_ms();std::function<void(int)> f[4];for (int X = 0; X < NH; ++X)f[X] = [&, X](int j) {@@ -5577,10 +5595,12 @@ struct Model::Impl {r0 - X * FFh, r1 - X * FFh, lim);};run_nodes(ne * GB, f);+ const double tp2 = now_ms();for (const Ent& en : ents)for (int i = 0; i < en.n; ++i)gg::iq_cpu_quant(fm.d_act, c_xhd.data() + (size_t) (en.slot0 + i) * FF, FF,aq + (size_t) (kWin + en.slot0 + i) * E * 4);+ const double tp3 = now_ms();for (int X = 0; X < NH; ++X)f[X] = [&, X](int j) {const Ent& en = ents[(size_t) (ord != nullptr ? ord[j / DB] : j / DB)];@@ -5594,6 +5614,16 @@ struct Model::Impl {gg::iq_cpu_down(fm, mx_part_ptr(l, en.e, 2, X), hp, en.n, yp, r0 - X * Eh, r1 - X * Eh);};run_nodes(ne * DB, f);+ { // DS4_PROF: where the CPU lane's time goes (act prep + prefetch, gate/up, h quant, down), summed+ static const bool on = std::getenv("DS4_PROF") != nullptr;+ if (on) {+ static double acc[6] = {};+ const double te = now_ms();+ acc[0] += tp1 - tp0; acc[1] += tp2 - tp1; acc[2] += tp3 - tp2; acc[3] += te - tp3; acc[4] += 1; acc[5] += ne;+ if (((long long) acc[4]) % 1000 == 0)+ std::fprintf(stderr, "ds4 prof CPU lane (IQ rows) per call: n %.0f, entries/call %.2f, prep %.3f ms, gate+up %.3f ms, h-quant %.3f ms, down %.3f ms\n", acc[4], acc[5] / acc[4], acc[0] / acc[4], acc[1] / acc[4], acc[2] / acc[4], acc[3] / acc[4]);+ }+ }}// the window's CPU-lane experts made ready (and pinned) in the store for the length of the calldiff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hppindex 28fe665be..9cac562c2 100644--- a/src/dsv41/ds4_model.hpp+++ b/src/dsv41/ds4_model.hpp@@ -42,6 +42,9 @@ struct Options {// dspark_auto: the drafter is loaded only where the routed experts fit the GPUs (more than 1.5x what they have, the verify rounds// read 3.3x the bytes of a token and measured no faster than plain decoding, and the drafter's 7 GB of experts take the VRAM the// trunk's hottest would have had); --dspark on / off / none says it outright+ // experts a word takes: -1 the rule (5 where the cards hold 16 GiB or less in all, else the model's 6), N the router's top N with+ // their weights renormalised (--experts 6: full quality); ds4_experts_rule.hpp+ int experts = -1;bool dspark_auto = true;bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decodingint dspark_block = 0; // at most this many drafts a round (--dspark-block N; 0: the pack's block size)diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cppnew file mode 100644index 000000000..f1fdc6d34--- /dev/null+++ b/tests/dsv41/experts_rule_test.cpp@@ -0,0 +1,68 @@+// Strata. The experts-per-word rule of the DeepSeek V4.1 engine (src/dsv41/ds4_experts_rule.hpp): 5 on a card with 16 GiB or less, 6+// elsewhere, --experts overrides, DS4_TOPK over both. No GPU needed.+#include "ds4_experts_rule.hpp"++#include <cstdio>+#include <cstdlib>++using strata::ds4::experts_per_word;+using strata::ds4::experts_test_override;++static int fails = 0;+#define CHECK_EQ(a, b) \+ do { \+ const long long x = (a), y = (b); \+ if (x != y) { \+ std::printf("FAIL %s:%d: %s = %lld, expected %lld\n", __FILE__, __LINE__, #a, x, y); \+ ++fails; \+ } \+ } while (0)++int main() {+ const uint64_t GiB = 1ull << 30;+ // the owner's 12 GB-class cards (RTX 5070 12227 MiB, RTX 3060 12 GB), 16 GiB cards: faster mode by default+ CHECK_EQ(experts_per_word(-1, 12227ull << 20, 6), 5);+ CHECK_EQ(experts_per_word(-1, 12 * GiB, 6), 5);+ CHECK_EQ(experts_per_word(-1, 16 * GiB, 6), 5);+ CHECK_EQ(experts_per_word(-1, 8 * GiB, 6), 5);+ // 24 GB and bigger, and several cards: the model's own 6+ CHECK_EQ(experts_per_word(-1, 16 * GiB + 1, 6), 6);+ CHECK_EQ(experts_per_word(-1, 24 * GiB, 6), 6);+ CHECK_EQ(experts_per_word(-1, 4 * 32 * GiB, 6), 6);+ CHECK_EQ(experts_per_word(-1, 2 * 12 * GiB, 6), 6); // two 12 GB cards hold 24 GiB in all+ // unknown memory (0): the model's count, never a silent cut+ CHECK_EQ(experts_per_word(-1, 0, 6), 6);+ // the flag decides, both ways+ CHECK_EQ(experts_per_word(6, 12 * GiB, 6), 6);+ CHECK_EQ(experts_per_word(4, 12 * GiB, 6), 4);+ CHECK_EQ(experts_per_word(5, 96 * GiB, 6), 5);+ CHECK_EQ(experts_per_word(9, 12 * GiB, 6), 6); // never more than the model has+ CHECK_EQ(experts_per_word(1, 12 * GiB, 6), 1);+ // a model that routes to 1 (or no) expert: left alone+ CHECK_EQ(experts_per_word(-1, 12 * GiB, 1), 1);+ CHECK_EQ(experts_per_word(-1, 12 * GiB, 0), 0);+ // a model that routes to fewer than 5: the rule never raises it+ CHECK_EQ(experts_per_word(-1, 12 * GiB, 4), 4);+ // the test override wins over the rule and the flag+#if defined(_WIN32)+ _putenv("DS4_TOPK=4");+#else+ setenv("DS4_TOPK", "4", 1);+#endif+ CHECK_EQ(experts_test_override(experts_per_word(-1, 96 * GiB, 6), 6), 4);+ CHECK_EQ(experts_test_override(experts_per_word(6, 12 * GiB, 6), 6), 4);+#if defined(_WIN32)+ _putenv("DS4_TOPK=9");+#else+ setenv("DS4_TOPK", "9", 1);+#endif+ CHECK_EQ(experts_test_override(5, 6), 5); // out of range: ignored+#if defined(_WIN32)+ _putenv("DS4_TOPK=");+#else+ unsetenv("DS4_TOPK");+#endif+ CHECK_EQ(experts_test_override(5, 6), 5);+ if (fails == 0) std::printf("experts rule: all checks passed\n");+ return fails == 0 ? 0 : 1;+}