Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: the chunked prompt's attention slab is automatic (DS4_ATTN_SLAB unset): up to 2048 rows where the bigger buffers take VRAM no expert would have used (the expert tier off, or every expert fits with room to spare), else the 0.2.6 rule; DS4_ATTN_SLAB=0 restores the 0.2.6 rule exactly, N rows overrides. If the automatic slab does not fit the load retries with the 256-row slab before it halves the chunk. Measured on the RTX 5070 (12 GB, 2.25 bpw): 1024 / 2048 rows gave +2% prompt but cost 29 / 83 VRAM experts, -3% decode and different greedy ids (placement), so there the rule keeps 256. attn_slab_rows() in ds4_experts_rule.hpp, 14 new checks in experts_rule_test.

Niko1221bdde697225d9
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hpp
index 60d5881c3..d5085a32f 100644
--- a/src/dsv41/ds4_experts_rule.hpp
+++ b/src/dsv41/ds4_experts_rule.hpp
@@ -7,6 +7,7 @@
#pragma once
#include <cstdint>
+#include <algorithm>
#include <cstdlib>
namespace strata::ds4 {
@@ -59,6 +60,24 @@ inline bool copy_all_fits(double lane_bytes, uint64_t ram_avail, double ram_marg
return lane_bytes + ram_margin <= (double) ram_avail && lane_bytes <= commit_room;
}
+// The chunked prompt's attention slab (rows a pass: the dense projections q_b / wo_a / wo_b are dequantized and multiplied once a slab, so
+// a 256-row slab dequantizes them 32 times for an 8K prompt). `base`: the 0.2.6 rule (8 MB of rows / heads width, 256 on this model).
+// DS4_ATTN_SLAB=N (`env` >= 0): 0 gives the 0.2.6 rule back, N > 0 asks for that many rows (64..chunk, a multiple of 64). Unset (`env` < 0):
+// automatic - up to 2048 rows, but only where the bigger buffers take VRAM no expert would have used: `spare_bytes` is the card's room
+// beyond what its expert tier can use (huge when the tier is off, e.g. --vram-experts 0), `bytes_per_row` the buffers' cost of one row.
+// The answers do not change with the slab (a row's arithmetic is its own), but the experts a card holds do: measured 2026-10-26 on the RTX
+// 5070 (12 GB, 2.25 bpw, 8K prompt): 1024 rows +2.0% prompt, 2048 rows +2.2%, at 29 / 83 fewer experts in VRAM, a -3% decode and different
+// greedy ids (placement); so a card whose room is all experts keeps the 0.2.6 slab, and the automatic size is for cards with VRAM to spare.
+inline int attn_slab_rows(int env, int chunk, int base, double spare_bytes, double bytes_per_row) {
+ auto clampn = [&](int a) { return std::max(64, std::min(chunk, a) / 64 * 64); };
+ if (env == 0) return base;
+ if (env > 0) return clampn(env);
+ const int target = clampn(2048);
+ if (target <= base) return base;
+ const double extra = (double) (target - base) * bytes_per_row;
+ return spare_bytes >= extra * 1.25 + (256.0 * 1048576.0) ? target : base;
+}
+
// DS4_TOPK=N (a test, not for users): over the flag and the rule; returns the count to use
inline int experts_test_override(int chosen, int model_k) {
if (const char* v = std::getenv("DS4_TOPK")) {
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index c926030ef..c2cb3c017 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -2670,6 +2670,14 @@ struct Model::Impl {
host_allocs.pop_back();
}
chunks.clear();
+ if (slab_auto_grew_) { // the automatic slab did not fit: the 0.2.6 slab first, the chunk only if that does not fit either
+ slab_auto_grew_ = false;
+ slab_force_base_ = true;
+ std::fprintf(stderr, "ds4: the automatic attention slab did not fit (%s): trying the 256-row slab
+", aerr.c_str());
+ aerr.clear();
+ continue;
+ }
const int was = chunk;
chunk = chunk > 512 ? chunk / 2 / 16 * 16 : 0;
if (chunk > 0) {
@@ -2960,7 +2968,9 @@ struct Model::Impl {
room / 1e9, other_b / 1e9, fit ? 16 : 400);
return fit ? 16 : 400;
}
+ bool slab_auto_grew_ = false, slab_force_base_ = false; // the automatic attention slab: grew this try / must not grow (it did not fit)
bool alloc_chunks(std::string& err) {
+ slab_auto_grew_ = false;
chunks.resize(devs.size());
const int D = g.dim, H = g.hc, HD = g.hd;
const int cols = (pk.eg_ngram - 1) * pk.eg_heads, ehd = pk.eg_hd;
@@ -2979,10 +2989,29 @@ struct Model::Impl {
// DS4_ATTN_SLAB=N: the attention slab's rows. The dense projections (q_b, wo_a/b) are dequantized and GEMM'd
// once a slab, so a slab of 256 rows dequantizes the same weights 32 times for an 8K prompt (measured: the
// dense dequant 916 ms of the 8K prompt, rocBLAS 867 ms); a larger slab costs VRAM for c.q/c.o and the oa
- // buffers (A * HQ x 4 bytes each).
- if (const char* v = std::getenv("DS4_ATTN_SLAB")) {
- const int a = std::atoi(v);
- if (a > 0) c.A = std::max(64, std::min(chunk, a) / 64 * 64);
+ // buffers (A * HQ x 4 bytes each). 0.2.7: unset = automatic (attn_slab_rows: up to 2048 rows where the buffers
+ // take VRAM no expert would have used), 0 = the 0.2.6 rule above, N = that many rows.
+ {
+ int env = -1;
+ if (const char* v = std::getenv("DS4_ATTN_SLAB")) env = std::max(0, std::atoi(v));
+ if (slab_force_base_ && env < 0) env = 0;
+ // per row: c.q + c.o (float), c.o16 (half), c.oa (float), c.oa16 (half)
+ const double bpr = (double) HQ * 10.0 + (double) OG * (double) g.o_lora * 6.0;
+ double spare = 1e18; // the expert tier off (--vram-experts 0): the buffers cost no expert
+ if (o.vram_experts > 0.0) {
+ double exp_b = 0, room = 0, other_b = 0;
+ gpu_expert_share(&exp_b, &room, &other_b);
+ spare = (room - other_b - exp_b) / (double) std::max<size_t>(1, devs.size());
+ }
+ const int base = c.A;
+ c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr);
+ if (env < 0 && c.A > base) {
+ slab_auto_grew_ = true;
+ if (di == 0 && o.verbose)
+ std::fprintf(stderr, "ds4: attention slab %d rows (automatic: %.1f GB of VRAM to spare beyond the experts; DS4_ATTN_SLAB=0 keeps %d)
+",
+ c.A, spare / 1e9, base);
+ }
}
// (the indexer's slab: its scores rows are the context wide - about 64 MB of them)
c.AI = std::max(32, std::min(128, (int) (((size_t) 64 << 20) / (((size_t) o.max_context + 64) * 4)) / 32 * 32));
diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cpp
index f0fad2dc5..ffc5078d4 100644
--- a/tests/dsv41/experts_rule_test.cpp
+++ b/tests/dsv41/experts_rule_test.cpp
@@ -8,6 +8,7 @@
using strata::ds4::experts_per_word;
using strata::ds4::experts_test_override;
using strata::ds4::copy_all_fits;
+using strata::ds4::attn_slab_rows;
static int fails = 0;
#define CHECK_EQ(a, b) \
@@ -70,6 +71,25 @@ int main() {
unsetenv("DS4_TOPK");
#endif
CHECK_EQ(experts_test_override(5, 6), 5);
+ // attention slab rows: attn_slab_rows(env, chunk, base, spare_bytes, bytes_per_row). bpr of this model ~ 32768 * 10 + 8 * 1024 * 6 = 377 KB.
+ {
+ const double bpr = 377000.0;
+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, 1e18, bpr), 2048); // unset, the expert tier is off (--vram-experts 0): automatic 2048
+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, 4.0e9, bpr), 2048); // 4 GB of VRAM to spare: automatic
+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, -5.0e9, bpr), 256); // the 5070: every spare byte is an expert's: the 0.2.6 slab
+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, 0.0, bpr), 256);
+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, 0.9e9, bpr), 256); // 1792 rows x 377 KB = 676 MB; needs 1.25 x + 256 MB = 1.1 GB
+ CHECK_EQ(attn_slab_rows(-1, 8192, 256, 1.2e9, bpr), 2048);
+ CHECK_EQ(attn_slab_rows(-1, 1024, 256, 1e18, bpr), 1024); // a 1024 chunk caps the slab
+ CHECK_EQ(attn_slab_rows(-1, 4096, 256, 1e18, bpr), 2048);
+ CHECK_EQ(attn_slab_rows(-1, 128, 64, 1e18, bpr), 128);
+ CHECK_EQ(attn_slab_rows(-1, 8192, 2560, 1e18, bpr), 2560); // a base already above the automatic size is kept
+ CHECK_EQ(attn_slab_rows(0, 8192, 256, 1e18, bpr), 256); // DS4_ATTN_SLAB=0: the 0.2.6 rule, whatever the spare
+ CHECK_EQ(attn_slab_rows(1024, 8192, 256, -5.0e9, bpr), 1024); // N: that many rows, the spare is the user's call
+ CHECK_EQ(attn_slab_rows(100, 8192, 256, 1e18, bpr), 64); // rounded down to 64, at least 64
+ CHECK_EQ(attn_slab_rows(99999, 8192, 256, 1e18, bpr), 8192); // at most the chunk
+ CHECK_EQ(attn_slab_rows(300, 8192, 256, 1e18, bpr), 256);
+ }
// prompt chunk: prefill_chunk_default(own_default, gpu_share, gguf_copy, file_experts, hip, min_vram, big=8192). What `strata launch` runs on one card
// passes neither --tp nor --threads, but the engine's default tp is 2, so own_default is false there; a launch with several cards passes --tp 1.
using strata::ds4::prefill_chunk_default;