Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: the chunked prompt's attention slab is automatic (DS4_ATTN_SLAB unset): up to 2048 rows where the bigger buffers take VRAM no expert would have used (the expert tier off, or every expert fits with room to spare), else the 0.2.6 rule; DS4_ATTN_SLAB=0 restores the 0.2.6 rule exactly, N rows overrides. If the automatic slab does not fit the load retries with the 256-row slab before it halves the chunk. Measured on the RTX 5070 (12 GB, 2.25 bpw): 1024 / 2048 rows gave +2% prompt but cost 29 / 83 VRAM experts, -3% decode and different greedy ids (placement), so there the rule keeps 256. attn_slab_rows() in ds4_experts_rule.hpp, 14 new checks in experts_rule_test.
Niko1221bdde697225d9
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hppindex 60d5881c3..d5085a32f 100644--- a/src/dsv41/ds4_experts_rule.hpp+++ b/src/dsv41/ds4_experts_rule.hpp@@ -7,6 +7,7 @@#pragma once#include <cstdint>+#include <algorithm>#include <cstdlib>namespace strata::ds4 {@@ -59,6 +60,24 @@ inline bool copy_all_fits(double lane_bytes, uint64_t ram_avail, double ram_margreturn lane_bytes + ram_margin <= (double) ram_avail && lane_bytes <= commit_room;}+// The chunked prompt's attention slab (rows a pass: the dense projections q_b / wo_a / wo_b are dequantized and multiplied once a slab, so+// a 256-row slab dequantizes them 32 times for an 8K prompt). `base`: the 0.2.6 rule (8 MB of rows / heads width, 256 on this model).+// DS4_ATTN_SLAB=N (`env` >= 0): 0 gives the 0.2.6 rule back, N > 0 asks for that many rows (64..chunk, a multiple of 64). Unset (`env` < 0):+// automatic - up to 2048 rows, but only where the bigger buffers take VRAM no expert would have used: `spare_bytes` is the card's room+// beyond what its expert tier can use (huge when the tier is off, e.g. --vram-experts 0), `bytes_per_row` the buffers' cost of one row.+// The answers do not change with the slab (a row's arithmetic is its own), but the experts a card holds do: measured 2026-10-26 on the RTX+// 5070 (12 GB, 2.25 bpw, 8K prompt): 1024 rows +2.0% prompt, 2048 rows +2.2%, at 29 / 83 fewer experts in VRAM, a -3% decode and different+// greedy ids (placement); so a card whose room is all experts keeps the 0.2.6 slab, and the automatic size is for cards with VRAM to spare.+inline int attn_slab_rows(int env, int chunk, int base, double spare_bytes, double bytes_per_row) {+ auto clampn = [&](int a) { return std::max(64, std::min(chunk, a) / 64 * 64); };+ if (env == 0) return base;+ if (env > 0) return clampn(env);+ const int target = clampn(2048);+ if (target <= base) return base;+ const double extra = (double) (target - base) * bytes_per_row;+ return spare_bytes >= extra * 1.25 + (256.0 * 1048576.0) ? target : base;+}+// DS4_TOPK=N (a test, not for users): over the flag and the rule; returns the count to useinline int experts_test_override(int chosen, int model_k) {if (const char* v = std::getenv("DS4_TOPK")) {diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex c926030ef..c2cb3c017 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -2670,6 +2670,14 @@ struct Model::Impl {host_allocs.pop_back();}chunks.clear();+ if (slab_auto_grew_) { // the automatic slab did not fit: the 0.2.6 slab first, the chunk only if that does not fit either+ slab_auto_grew_ = false;+ slab_force_base_ = true;+ std::fprintf(stderr, "ds4: the automatic attention slab did not fit (%s): trying the 256-row slab+", aerr.c_str());+ aerr.clear();+ continue;+ }const int was = chunk;chunk = chunk > 512 ? chunk / 2 / 16 * 16 : 0;if (chunk > 0) {@@ -2960,7 +2968,9 @@ struct Model::Impl {room / 1e9, other_b / 1e9, fit ? 16 : 400);return fit ? 16 : 400;}+ bool slab_auto_grew_ = false, slab_force_base_ = false; // the automatic attention slab: grew this try / must not grow (it did not fit)bool alloc_chunks(std::string& err) {+ slab_auto_grew_ = false;chunks.resize(devs.size());const int D = g.dim, H = g.hc, HD = g.hd;const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;@@ -2979,10 +2989,29 @@ struct Model::Impl {// DS4_ATTN_SLAB=N: the attention slab's rows. The dense projections (q_b, wo_a/b) are dequantized and GEMM'd// once a slab, so a slab of 256 rows dequantizes the same weights 32 times for an 8K prompt (measured: the// dense dequant 916 ms of the 8K prompt, rocBLAS 867 ms); a larger slab costs VRAM for c.q/c.o and the oa- // buffers (A * HQ x 4 bytes each).- if (const char* v = std::getenv("DS4_ATTN_SLAB")) {- const int a = std::atoi(v);- if (a > 0) c.A = std::max(64, std::min(chunk, a) / 64 * 64);+ // buffers (A * HQ x 4 bytes each). 0.2.7: unset = automatic (attn_slab_rows: up to 2048 rows where the buffers+ // take VRAM no expert would have used), 0 = the 0.2.6 rule above, N = that many rows.+ {+ int env = -1;+ if (const char* v = std::getenv("DS4_ATTN_SLAB")) env = std::max(0, std::atoi(v));+ if (slab_force_base_ && env < 0) env = 0;+ // per row: c.q + c.o (float), c.o16 (half), c.oa (float), c.oa16 (half)+ const double bpr = (double) HQ * 10.0 + (double) OG * (double) g.o_lora * 6.0;+ double spare = 1e18; // the expert tier off (--vram-experts 0): the buffers cost no expert+ if (o.vram_experts > 0.0) {+ double exp_b = 0, room = 0, other_b = 0;+ gpu_expert_share(&exp_b, &room, &other_b);+ spare = (room - other_b - exp_b) / (double) std::max<size_t>(1, devs.size());+ }+ const int base = c.A;+ c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr);+ if (env < 0 && c.A > base) {+ slab_auto_grew_ = true;+ if (di == 0 && o.verbose)+ std::fprintf(stderr, "ds4: attention slab %d rows (automatic: %.1f GB of VRAM to spare beyond the experts; DS4_ATTN_SLAB=0 keeps %d)+",+ c.A, spare / 1e9, base);+ }}// (the indexer's slab: its scores rows are the context wide - about 64 MB of them)c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cppindex f0fad2dc5..ffc5078d4 100644--- a/tests/dsv41/experts_rule_test.cpp+++ b/tests/dsv41/experts_rule_test.cpp@@ -8,6 +8,7 @@using strata::ds4::experts_per_word;using strata::ds4::experts_test_override;using strata::ds4::copy_all_fits;+using strata::ds4::attn_slab_rows;static int fails = 0;#define CHECK_EQ(a, b) \@@ -70,6 +71,25 @@ int main() {unsetenv("DS4_TOPK");#endifCHECK_EQ(experts_test_override(5, 6), 5);+ // attention slab rows: attn_slab_rows(env, chunk, base, spare_bytes, bytes_per_row). bpr of this model ~ 32768 * 10 + 8 * 1024 * 6 = 377 KB.+ {+ const double bpr = 377000.0;+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, 1e18, bpr), 2048); // unset, the expert tier is off (--vram-experts 0): automatic 2048+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, 4.0e9, bpr), 2048); // 4 GB of VRAM to spare: automatic+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, -5.0e9, bpr), 256); // the 5070: every spare byte is an expert's: the 0.2.6 slab+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, 0.0, bpr), 256);+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, 0.9e9, bpr), 256); // 1792 rows x 377 KB = 676 MB; needs 1.25 x + 256 MB = 1.1 GB+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, 1.2e9, bpr), 2048);+ CHECK_EQ(attn_slab_rows(-1, 1024, 256, 1e18, bpr), 1024); // a 1024 chunk caps the slab+ CHECK_EQ(attn_slab_rows(-1, 4096, 256, 1e18, bpr), 2048);+ CHECK_EQ(attn_slab_rows(-1, 128, 64, 1e18, bpr), 128);+ CHECK_EQ(attn_slab_rows(-1, 8192, 2560, 1e18, bpr), 2560); // a base already above the automatic size is kept+ CHECK_EQ(attn_slab_rows(0, 8192, 256, 1e18, bpr), 256); // DS4_ATTN_SLAB=0: the 0.2.6 rule, whatever the spare+ CHECK_EQ(attn_slab_rows(1024, 8192, 256, -5.0e9, bpr), 1024); // N: that many rows, the spare is the user's call+ CHECK_EQ(attn_slab_rows(100, 8192, 256, 1e18, bpr), 64); // rounded down to 64, at least 64+ CHECK_EQ(attn_slab_rows(99999, 8192, 256, 1e18, bpr), 8192); // at most the chunk+ CHECK_EQ(attn_slab_rows(300, 8192, 256, 1e18, bpr), 256);+ }// prompt chunk: prefill_chunk_default(own_default, gpu_share, gguf_copy, file_experts, hip, min_vram, big=8192). What `strata launch` runs on one card// passes neither --tp nor --threads, but the engine's default tp is 2, so own_default is false there; a launch with several cards passes --tp 1.using strata::ds4::prefill_chunk_default;