Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: the experts-read-in-place default prompt chunk is 4096 on non-HIP cards under 16 GiB (was 8192): decode +2% agent / +0.4..9% code (5 pairs on RTX 5070 and RTX 3060, 120 more experts resident), an agent turn of 3K new tokens on 16K takes the same time, a cold 8K prompt 9% slower; DS4_FEEDLOG prints each feed's time; test rows for 12 GB, 15.9 / 16 / 24 GiB, 8 GB, unknown VRAM and HIP
Niko12211fac68d6f288
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hppindex 33ca78772..d92d41581 100644--- a/src/dsv41/ds4_experts_rule.hpp+++ b/src/dsv41/ds4_experts_rule.hpp@@ -33,6 +33,7 @@ inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k) {// Reference KL against the original model does not move with the chunk (1024 0.1705, 2048 0.1692, 4096 0.1688, 8192 0.1695).// Alloc failure halves the chunk at load (4096 -> 2048 -> 1024 ...), KV and score buffers are allocated whole at the full --max-context.constexpr uint64_t kChunk4096MinVram = 10ull << 30;+constexpr uint64_t kChunkInPlace4096Below = 16ull << 30;// own_default: a single-card run (no --threads, tp 1); gpu_share: the share of the routed experts the cards can hold (1 = all);// gguf_copy: a GGUF pack with --mmap-experts off; file_experts: the experts stay in the files; hip: a HIP build; min_vram_bytes: the// smallest card; big: the chunk of the streamed case (the engine's kVlRows, 8192)@@ -40,6 +41,11 @@ inline int prefill_chunk_default(bool own_default, double gpu_share, bool gguf_cint big = 8192) {if (own_default && gpu_share >= 0.5) return 1024;if (gguf_copy && !hip && min_vram_bytes >= kChunk4096MinVram) return 4096;+ // the experts read in place on a card under 16 GiB (the 12 GB PCs; not measured on HIP): 4096 keeps ~120 more experts resident than 8192 (RTX 5070+ // 287 against 165, RTX 3060 386 against 277) - 5 interleaved pairs, 4096 against 8192, decode: 5070 agent x1.020 (5/5) code x1.004 (3/5), 3060+ // agent x1.018 (4/5) code x1.091 (5/5); an agent turn of 3K new tokens on a 16K context takes the same time (5070 18.9-19.2 s against 19.1-19.4 s,+ // 3060 21.4 s against 20.9-21.4 s) - for 9% less speed on a cold 8K prompt (5070 223 against 244 tok/s)+ if (file_experts && !hip && min_vram_bytes != 0 && min_vram_bytes < kChunkInPlace4096Below) return 4096;return file_experts ? big : 2048;}diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex 857df4128..5e2f68882 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -1364,10 +1364,13 @@ int main(int argc, char** argv) {for (size_t a = 0; a < ids.size();) {const int T = (int) std::min<size_t>((size_t) window, ids.size() - a);const bool last = a + (size_t) T == ids.size();+ const double tf0 = now_ms();if (!m.feed(ids.data() + a, T, df != nullptr ? T : (last ? 1 : 0), err)) {std::fprintf(stderr, "maya-ds4: %s\n", err.c_str());return 1;}+ if (std::getenv("DS4_FEEDLOG") != nullptr) // each feed's time: the incremental turn (--window W: a long prefix, then a short one)+ std::fprintf(stderr, "feed at %zu: %d tokens in %.0f ms (%.1f tok/s)\n", a, T, now_ms() - tf0, 1000.0 * T / (now_ms() - tf0));if (df != nullptr && m.logits(lg)) std::fwrite(lg.data(), 4, lg.size(), df);a += (size_t) T;}diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 8e5826928..a9a0ab30b 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -1983,7 +1983,7 @@ struct Model::Impl {#endifconst int rule_chunk = prefill_chunk_default(own_default, gpu_expert_share(), gguf_copy, file_experts, hip_build, vmin_bytes, kVlRows);chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : rule_chunk) : 0;- if (prompt_par && o.prefill_chunk < 0 && rule_chunk == 4096)+ if (prompt_par && o.prefill_chunk < 0 && rule_chunk == 4096 && gguf_copy)std::fprintf(stderr, "ds4: prompts in chunks of 4096 tokens (the experts are copied into RAM, %.0f GiB of graphics memory; --prefill-chunk N changes it)\n",(double) vmin_bytes / (double) (1ull << 30));const char* dc = std::getenv("DS4_CHUNK"); // (tests: down to 16, a short prompt in several chunks)diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cppindex dec47073f..1d1810734 100644--- a/tests/dsv41/experts_rule_test.cpp+++ b/tests/dsv41/experts_rule_test.cpp@@ -66,11 +66,19 @@ int main() {// prompt chunk: prefill_chunk_default(own_default, gpu_share, gguf_copy, file_experts, hip, min_vram, big=8192). What `strata launch` runs on one card// passes neither --tp nor --threads, but the engine's default tp is 2, so own_default is false there; a launch with several cards passes --tp 1.using strata::ds4::prefill_chunk_default;- // RTX 5070 / RTX 3060 (12 GB), launch default: experts in place -> 8192 (their logs say "prompts in chunks of 8192")- CHECK_EQ(prefill_chunk_default(false, 0.06, false, true, false, 12227ull << 20), 8192);- CHECK_EQ(prefill_chunk_default(false, 0.06, false, true, false, 12 * GiB), 8192);- // the same cards with --tp 1 on one card: share 0.06 < 0.5 -> 8192 as well (the streamed case)- CHECK_EQ(prefill_chunk_default(true, 0.06, false, true, false, 12 * GiB), 8192);+ // RTX 5070 / RTX 3060 (12 GB), launch default: experts in place on a card under 16 GiB -> 4096 (their logs said 8192 before this rule)+ CHECK_EQ(prefill_chunk_default(false, 0.06, false, true, false, 12227ull << 20), 4096);+ CHECK_EQ(prefill_chunk_default(false, 0.06, false, true, false, 12 * GiB), 4096);+ // the same cards with --tp 1 on one card: share 0.06 < 0.5 -> the same 4096+ CHECK_EQ(prefill_chunk_default(true, 0.06, false, true, false, 12 * GiB), 4096);+ // 16 GB cards (15.9 GiB total) 4096; a card with exactly 16 GiB or more keeps the big chunk; short VRAM (8 GB) 4096; unknown VRAM the big one+ CHECK_EQ(prefill_chunk_default(false, 0.10, false, true, false, 16 * GiB - 1), 4096);+ CHECK_EQ(prefill_chunk_default(false, 0.10, false, true, false, 16 * GiB), 8192);+ CHECK_EQ(prefill_chunk_default(false, 0.10, false, true, false, 24 * GiB), 8192);+ CHECK_EQ(prefill_chunk_default(false, 0.0, false, true, false, 8 * GiB), 4096);+ CHECK_EQ(prefill_chunk_default(false, 0.0, false, true, false, 0), 8192);+ // a 12 GB HIP card is not measured: the big chunk+ CHECK_EQ(prefill_chunk_default(false, 0.06, false, true, true, 12 * GiB), 8192);// a 5090-class card (32 GB, share 0.19): in place 8192; with the experts copied into RAM 4096CHECK_EQ(prefill_chunk_default(false, 0.19, false, true, false, 32 * GiB), 8192);CHECK_EQ(prefill_chunk_default(false, 0.19, true, true, false, 32 * GiB), 4096);@@ -78,8 +86,8 @@ int main() {CHECK_EQ(prefill_chunk_default(false, 0.10, true, true, false, 16 * GiB), 4096);CHECK_EQ(prefill_chunk_default(false, 0.05, true, true, false, 24 * GiB), 4096);CHECK_EQ(prefill_chunk_default(false, 0.06, true, true, false, 12227ull << 20), 4096);- // copied into RAM on a card under 10 GiB (short VRAM): the streamed default, never the 4096 rule- CHECK_EQ(prefill_chunk_default(false, 0.0, true, true, false, 8 * GiB), 8192);+ // copied into RAM on a card under 10 GiB (short VRAM): not the copy rule; the in-place rule for a small card gives 4096, unknown VRAM 8192+ CHECK_EQ(prefill_chunk_default(false, 0.0, true, true, false, 8 * GiB), 4096);CHECK_EQ(prefill_chunk_default(false, 0.0, true, true, false, 0), 8192);// 1 x R9700 (HIP, 32 GB, share 0.19), single-card default run (tp 1 passed by a multi-card box): 8192, also when copied (HIP is not measured at 4096)CHECK_EQ(prefill_chunk_default(true, 0.19, false, true, true, 32 * GiB), 8192);