Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek: the 4 x R9700 share-based defaults and the drafter cut apply when every device is RDNA4 (gfx1200/gfx1201), not to every HIP build (Impl::rdna4)
Niko1221164619c5a84f
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hppindex 746822ea7..140b3df4a 100644--- a/src/dsv41/ds4_experts_rule.hpp+++ b/src/dsv41/ds4_experts_rule.hpp@@ -47,7 +47,7 @@ inline int prefill_chunk_default(bool own_default, double gpu_share, bool gguf_c// `strata launch` writes for 4 or more AMD cards), share 0.86: 1024 370 against 214 tok/s at 8192; and with the engine's own default tp// (2, pairs of cards, nothing written), where the rule used to fall through to 2048/8192: 2.25 bpw, --dspark auto, ids 5705e1164a, 8K// warm prompts 1024 -> 1247 ppts against 690 at 8192, decode unchanged. So a multi-card tp run takes it too. One card with the default- // tp, `--threads on` (it stalled at the first 8K prompt in the 4-card runs) and a build that is not HIP keep their old defaults: not measured with 1024.+ // tp, `--threads on` (it stalled at the first 8K prompt in the 4-card runs) and a system that is not RDNA4 keep their old defaults: not measured with 1024.// An explicit --prefill-chunk N wins over all of this (the call site takes the rule only when none was given).if ((own_default || multi_tp) && gpu_share >= 0.5) return 1024;if (gguf_copy && !hip && min_vram_bytes >= kChunk4096MinVram) return 4096;@@ -108,7 +108,7 @@ inline int attn_slab_rows(int env, int chunk, int base, double spare_bytes, doub// summed room, 64 MB a GB, 400..1200 MB. Measured on Strix Halo only (400 -> 1200: the queue 3,200 -> 8,087 experts, 4K prompt 27.3// -> 21.3 s, decode unchanged); a discrete card in this band is not measured and takes the same, VRAM-bounded, rule.// A unified-memory GPU never takes the small ring (its datapoint is the 1200 MB one): it keeps the 16 MB "fit" rule at 1/1.3.-// `hip`: the 0.5..0.82 band and the 0.82 threshold are the 4 x R9700 measurements, so only a HIP build takes them; any other build keeps main's+// `hip`: the 0.5..0.82 band and the 0.82 threshold are the 4 x R9700 measurements, so only an RDNA4 system (every device a gfx1200 or gfx1201; the parameter is named hip) takes them; any other build keeps main's// rule there (16 MB from 1/1.3, the streamed ring below it) until a CUDA multi-card rig is measured.inline int staging_ring_mb(double share, double room_bytes, int n_dev, bool integrated, bool hip) {const double GB = 1e9;diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex d35398e21..e36ed7673 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -78,8 +78,7 @@ namespace gg = strata::ds4::gguf;namespace {-// The DeepSeek engine's share-based defaults (small chunk, whole-chunk slab, small ring, pinned copy) are the 4 x R9700 measurements: only a HIP-// build takes them.+// A HIP build (the share-based defaults below are gated on the devices being RDNA4, Impl::rdna4()).#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)constexpr bool kHipBuild = true;#else@@ -819,7 +818,25 @@ struct Model::Impl {std::vector<Group> groups;std::vector<std::unique_ptr<Worker>> workers; // threads mode: one per devicebool par = false; // threads mode (the windows: decode)- mutable int all_gfx12 = -1; // gpus_are_gfx12(): -1 not asked yet+ mutable int all_gfx12 = -1; // rdna4(): -1 not asked yet+ // Every GPU the engine uses is a gfx1200 / gfx1201 (RDNA4: R9700, RX 9000) in a HIP build. The DeepSeek engine's share-based defaults (the small+ // chunk for multi-card tp runs, the whole-chunk slab, the small staging ring, the pinned copy at share >= 0.5) and the drafter's cumulative cut are+ // the 4 x R9700 measurements, so only RDNA4 takes them until another AMD part is tested (Strix Halo next).+ bool rdna4() const {+ if (all_gfx12 < 0) {+ bool yes = false;+#if defined(STRATA_USE_HIP)+ yes = !devs.empty();+ for (const auto& d : devs) {+ cudaDeviceProp prop{};+ yes = yes && cudaGetDeviceProperties(&prop, d.id) == cudaSuccess &&+ (std::strncmp(prop.gcnArchName, "gfx1200", 7) == 0 || std::strncmp(prop.gcnArchName, "gfx1201", 7) == 0);+ }+#endif+ all_gfx12 = yes ? 1 : 0;+ }+ return all_gfx12 == 1;+ }bool prompt_par = false; // the prompt path's chunks (a thread a GPU), as --tp / --threads askedunsigned win_seq = 0; // the windows the threads ran (the hand-offs' seq)// the chunked prompt path (threads mode): chunk tokens a pass, runs shorter than chunk_min in windows@@ -2040,7 +2057,7 @@ struct Model::Impl {// most (4 x R9700: 8192 214, 1024 370) the small one wins; with the experts copied into RAM (--mmap-experts off) 4096 beats 8192 by 7-9%// (H100 32 GiB: 2048 502, 4096 538, 8192 495). A GGUF pack always has file_experts set (copied only by --mmap-experts off).const bool own_default = !(par || o.tp > 1); // (a run with tp 1 and no --threads: `strata launch` writes --tp 1 for 4+ AMD cards, --tp 2 for 2-3 and nothing for one card = the engine's tp 2)- const bool multi_tp = kHipBuild && !own_default && !o.threads && devs.size() >= 2; // (the 4-card default-tp run the 1024 chunk was also measured on; an explicit --prefill-chunk below wins over the rule)+ const bool multi_tp = rdna4() && !own_default && !o.threads && devs.size() >= 2; // (the 4-card default-tp run the 1024 chunk was also measured on; an explicit --prefill-chunk below wins over the rule)const bool gguf_copy = pk.gguf && (o.mmap_experts == 0 || (store_gb() <= 0.0 && auto_copy_all));uint64_t vmin_bytes = ~0ull;for (const Dev& dv : devs) vmin_bytes = std::min<uint64_t>(vmin_bytes, (uint64_t) dv.total);@@ -2983,7 +3000,7 @@ struct Model::Impl {cudaDeviceProp ip{};if (cudaGetDeviceProperties(&ip, d.id) == cudaSuccess && ip.integrated != 0) unified = true;}- const size_t mb = (size_t) strata::ds4::staging_ring_mb(exp_b > 0 ? share : 0.0, room, (int) devs.size(), unified, kHipBuild);+ const size_t mb = (size_t) strata::ds4::staging_ring_mb(exp_b > 0 ? share : 0.0, room, (int) devs.size(), unified, rdna4());if (o.verbose)std::fprintf(stderr, "ds4: staging ring: experts %.1f GB, VRAM room %.1f GB, dense %.1f GB -> %d MB\n", exp_b / 1e9,room / 1e9, other_b / 1e9, (int) mb);@@ -3043,11 +3060,11 @@ struct Model::Impl {const int base = c.A;// one rule (ds4_experts_rule.hpp: attn_slab_rows): the chunk's whole slab (<= 1024 rows) where the cards hold half the// experts or more (4 x R9700, +9% prompt), up to 2048 rows where the VRAM is spare, the base otherwise- c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr, kHipBuild ? share : 0.0);+ c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr, rdna4() ? share : 0.0);if (env < 0 && c.A > base) {slab_auto_grew_ = true;if (di == 0 && o.verbose) {- if (kHipBuild && share >= 0.5)+ if (rdna4() && share >= 0.5)std::fprintf(stderr, "ds4: attention slab %d rows (automatic: the cards hold %.0f%% of the experts; DS4_ATTN_SLAB=0 keeps %d)\n",c.A, share * 100.0, base);else@@ -4599,10 +4616,10 @@ struct Model::Impl {// DS4_COPY_PIN_RO=1: cudaHostRegisterReadOnly (the GPU only reads it).// 0.2.7: DS4_COPY_PIN unset = ON when EITHER condition measured reliable holds: (a) the whole arena is on explicit 2 MB pages (a pin of a// THP arena registers 4 KB pages and the DMA rate was erratic on the 12 GB cell: 11.9 s one prompt, 30.4 s the next), or (b) the- // cards hold at least half the routed experts in a HIP build (4 x R9700: the pin was reliable in 4/4 runs there, see above). =0 is the off+ // cards hold at least half the routed experts on RDNA4 (4 x R9700: the pin was reliable in 4/4 runs there, see above). =0 is the off// switch, =1 pins any arena.const char* cp = std::getenv("DS4_COPY_PIN");- const bool pin_copies = cp != nullptr ? std::atoi(cp) != 0 : ((arena_all_huge || (kHipBuild && gpu_expert_share() >= 0.5)) && !ram_chunks.empty());+ const bool pin_copies = cp != nullptr ? std::atoi(cp) != 0 : ((arena_all_huge || (rdna4() && gpu_expert_share() >= 0.5)) && !ram_chunks.empty());if (pin_copies) {const bool ro = std::getenv("DS4_COPY_PIN_RO") != nullptr && std::atoi(std::getenv("DS4_COPY_PIN_RO")) != 0;const double tr = now_ms();@@ -9672,21 +9689,7 @@ void Model::set_images(std::vector<Image> images) {p_->imgs = std::move(images);}int Model::draft_block() const { return p_->g.n_mtp > 0 ? p_->g.d_block : 0; }-bool Model::gpus_are_gfx12() const {- if (p_->all_gfx12 < 0) {- bool yes = false;-#if defined(STRATA_USE_HIP)- yes = !p_->devs.empty();- for (const auto& d : p_->devs) {- cudaDeviceProp prop{};- yes = yes && cudaGetDeviceProperties(&prop, d.id) == cudaSuccess &&- (std::strncmp(prop.gcnArchName, "gfx1200", 7) == 0 || std::strncmp(prop.gcnArchName, "gfx1201", 7) == 0);- }-#endif- p_->all_gfx12 = yes ? 1 : 0;- }- return p_->all_gfx12 == 1;-}+bool Model::gpus_are_gfx12() const { return p_->rdna4(); }bool Model::draft(int y, std::vector<int>& d, std::vector<float>& c, std::string& err, float t, float tp, int tk,float mp, uint64_t seed, uint64_t counter) {return p_->draft(y, d, c, err, t, tp, tk, mp, seed, counter);diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cppindex fdaf50fef..1f9bfee73 100644--- a/tests/dsv41/experts_rule_test.cpp+++ b/tests/dsv41/experts_rule_test.cpp@@ -134,7 +134,7 @@ int main() {CHECK_EQ(staging_ring_mb(0.76, 40 * GB, 1, true, true), 1200);// nothing known: no devicesCHECK_EQ(staging_ring_mb(0.0, 0.0, 0, false, true), 400);- // a build that is not HIP keeps main's rule everywhere: 16 MB from 1/1.3, the streamed ring below it (no small band, no 0.82)+ // a system that is not RDNA4 keeps main's rule everywhere: 16 MB from 1/1.3, the streamed ring below it (no small band, no 0.82)CHECK_EQ(staging_ring_mb(0.77, 4 * 31 * GB, 4, false, false), 16);CHECK_EQ(staging_ring_mb(0.60, 4 * 31 * GB, 4, false, false), 1200);CHECK_EQ(staging_ring_mb(0.80, 4 * 31 * GB, 4, false, false), 16);@@ -174,7 +174,7 @@ int main() {CHECK_EQ(prefill_chunk_default(true, 0.86, false, true, true, 32 * GiB), 1024);CHECK_EQ(prefill_chunk_default(true, 0.50, false, true, true, 32 * GiB), 1024);CHECK_EQ(prefill_chunk_default(true, 0.49, false, true, true, 32 * GiB), 8192);- // a multi-card tp run (own_default false, multi_tp: 2+ cards, no --threads on; the engine's default tp 2, or --tp 2 from the launcher) takes+ // a multi-card tp run (own_default false, multi_tp: RDNA4, 2+ cards, no --threads on; the engine's default tp 2, or --tp 2 from the launcher) takes// the small chunk too: the 4 x R9700 default-tp datapoint (8K warm 1247 ppts at 1024 against 690)CHECK_EQ(prefill_chunk_default(false, 0.86, false, true, true, 32 * GiB, 8192, true), 1024);CHECK_EQ(prefill_chunk_default(false, 0.50, false, true, true, 32 * GiB, 8192, true), 1024);