Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: the prompt chunk of the experts-copied-into-RAM case (2048) and of an explicit --tp 1 single card (1024) becomes 4096 on cards with 10 GiB or more (HIP builds and smaller cards keep theirs; the in-place 8192 is unchanged); startup line; rule test incl. 12/16/24/32 GB, short-VRAM, HIP and max-context cases
Niko1221dba7d0e0ab77
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hppindex 316b6cdaa..6b0ee4ce1 100644--- a/src/dsv41/ds4_experts_rule.hpp+++ b/src/dsv41/ds4_experts_rule.hpp@@ -24,6 +24,19 @@ inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k) {return n < 1 ? 1 : n;}+// Prompt chunk (--prefill-chunk) of the single-card default, decided once at load from bytes (no timing). The chunk's buffers come out of the VRAM+// the expert tier would have: 1024 -> 4096 tokens takes 83 of 469 resident experts on a 12 GB card, which cost nothing measurable in decode+// (RTX 3060: x0.995 code, x1.002 agent) and made 8K prompts 20% faster (RTX 3060 175 -> 210, RTX 5070 186 -> 223, 32 GB class 285 -> 353 tok/s);+// 8192 returns up to 4% of agent decode. Cards under 10 GiB keep 1024 (their tier is the first thing a bigger buffer would eat).+constexpr uint64_t kChunk4096MinVram = 10ull << 30;+// fallback: the value the case had before (1024: an explicit --tp 1 single card; 2048: the experts copied into RAM; the experts read in place+// keep 8192, which is not decided here). hip: a HIP build keeps the fallback until a one-card Radeon sweep shows 4096 is no slower.+// The result does not depend on --max-context: the chunk's buffers and the KV are allocated whole at load for the full context, and a+// chunk the card cannot hold is halved there (4096 -> 2048 -> 1024 -> ...), so a session never runs out later.+inline int prefill_chunk_default(uint64_t min_vram_bytes, bool hip = false, int fallback = 1024) {+ return !hip && min_vram_bytes >= kChunk4096MinVram ? 4096 : fallback;+}+// DS4_TOPK=N (a test, not for users): over the flag and the rule; returns the count to useinline int experts_test_override(int chosen, int model_k) {if (const char* v = std::getenv("DS4_TOPK")) {diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 2c3ec87b4..614991840 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -1967,7 +1967,25 @@ struct Model::Impl {file_experts = exp_b - std::max(0.0, room - other_b) + (double) kRamMargin > (double) ram_avail();}const bool own_default = !(par || o.tp > 1); // (the single-card default run the chunked path newly takes: chunks of 1024 - 8K prompt 370 tok/s on 4 x R9700, 214 at 8192; the threads / tp runs keep theirs)- chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : own_default ? 1024 : file_experts ? kVlRows : 2048) : 0;+ // the chunk of the cases that are not the experts-read-in-place one (8192, measured best there): an explicit --tp 1 single card+ // (was 1024) and the experts copied into RAM (was 2048) take 4096 from 10 GiB of VRAM (ds4_experts_rule.hpp)+ int own_chunk = 1024, copy_chunk = 2048;+ if (!devs.empty() && (own_default || !file_experts)) {+ uint64_t vmin = ~0ull;+ for (const Dev& dv : devs) vmin = std::min<uint64_t>(vmin, (uint64_t) dv.total);+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)+ const bool hip_build = true;+#else+ const bool hip_build = false;+#endif+ own_chunk = prefill_chunk_default(vmin, hip_build, 1024);+ copy_chunk = prefill_chunk_default(vmin, hip_build, 2048);+ const int shown = own_default ? own_chunk : copy_chunk;+ if (prompt_par && o.prefill_chunk < 0 && shown == 4096)+ std::fprintf(stderr, "ds4: prompts in chunks of %d tokens (%.0f GiB of graphics memory: bigger chunks read the experts fewer times; "+ "--prefill-chunk N changes it)\n", shown, (double) vmin / (double) (1ull << 30));+ }+ chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : own_default ? own_chunk : file_experts ? kVlRows : copy_chunk) : 0;const char* dc = std::getenv("DS4_CHUNK"); // (tests: down to 16, a short prompt in several chunks)if (dc != nullptr) chunk = prompt_par ? std::atoi(dc) : 0;if (const char* v = std::getenv("DS4_CHUNK_MIN")) chunk_min = std::max(1, std::atoi(v));diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cppindex f1fdc6d34..73264141d 100644--- a/tests/dsv41/experts_rule_test.cpp+++ b/tests/dsv41/experts_rule_test.cpp@@ -63,6 +63,30 @@ int main() {unsetenv("DS4_TOPK");#endifCHECK_EQ(experts_test_override(5, 6), 5);+ // prompt chunk: 4096 from 10 GiB of VRAM (12 GB cards, 16, 24, 32, 80 GB), 1024 below+ using strata::ds4::prefill_chunk_default;+ CHECK_EQ(prefill_chunk_default(12227ull << 20), 4096);+ CHECK_EQ(prefill_chunk_default(12 * GiB), 4096);+ CHECK_EQ(prefill_chunk_default(32 * GiB), 4096);+ CHECK_EQ(prefill_chunk_default(80 * GiB), 4096);+ CHECK_EQ(prefill_chunk_default(10 * GiB), 4096);+ CHECK_EQ(prefill_chunk_default(10 * GiB - 1), 1024);+ CHECK_EQ(prefill_chunk_default(8 * GiB), 1024);+ CHECK_EQ(prefill_chunk_default(0), 1024);+ CHECK_EQ(prefill_chunk_default(16 * GiB), 4096);+ CHECK_EQ(prefill_chunk_default(24 * GiB), 4096);+ CHECK_EQ(prefill_chunk_default(12 * GiB, true), 1024); // HIP builds: 1024 until measured+ CHECK_EQ(prefill_chunk_default(32 * GiB, true), 1024);+ // the experts-copied-into-RAM case (fallback 2048): 4096 on 12/16/24/32 GB cards, 2048 below 10 GiB and on HIP builds+ CHECK_EQ(prefill_chunk_default(12227ull << 20, false, 2048), 4096);+ CHECK_EQ(prefill_chunk_default(16 * GiB, false, 2048), 4096);+ CHECK_EQ(prefill_chunk_default(24 * GiB, false, 2048), 4096);+ CHECK_EQ(prefill_chunk_default(32 * GiB, false, 2048), 4096);+ CHECK_EQ(prefill_chunk_default(8 * GiB, false, 2048), 2048);+ CHECK_EQ(prefill_chunk_default(0, false, 2048), 2048);+ CHECK_EQ(prefill_chunk_default(32 * GiB, true, 2048), 2048);+ // the rule takes no --max-context: a 128K session on a 12 GB card gets the same 4096, because the chunk's buffers and the KV are allocated+ // whole at load for that context and a chunk that does not fit there is halved before the first prompt (4096 -> 2048 -> 1024 -> ...)if (fails == 0) std::printf("experts rule: all checks passed\n");return fails == 0 ? 0 : 1;}