Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek: the 4 x R9700 share-based defaults (small chunk for multi-card tp runs, whole-chunk slab, small staging ring, pinned copy at share >= 0.5) apply to HIP builds only; the staging ring takes a hip flag; tests for both branches
Niko1221484c6893375c
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hppindex 50e23d98f..746822ea7 100644--- a/src/dsv41/ds4_experts_rule.hpp+++ b/src/dsv41/ds4_experts_rule.hpp@@ -47,7 +47,7 @@ inline int prefill_chunk_default(bool own_default, double gpu_share, bool gguf_c// `strata launch` writes for 4 or more AMD cards), share 0.86: 1024 370 against 214 tok/s at 8192; and with the engine's own default tp// (2, pairs of cards, nothing written), where the rule used to fall through to 2048/8192: 2.25 bpw, --dspark auto, ids 5705e1164a, 8K// warm prompts 1024 -> 1247 ppts against 690 at 8192, decode unchanged. So a multi-card tp run takes it too. One card with the default- // tp, and `--threads on` (it stalled at the first 8K prompt in the 4-card runs), keep their old defaults: not measured with 1024.+ // tp, `--threads on` (it stalled at the first 8K prompt in the 4-card runs) and a build that is not HIP keep their old defaults: not measured with 1024.// An explicit --prefill-chunk N wins over all of this (the call site takes the rule only when none was given).if ((own_default || multi_tp) && gpu_share >= 0.5) return 1024;if (gguf_copy && !hip && min_vram_bytes >= kChunk4096MinVram) return 4096;@@ -81,7 +81,8 @@ inline bool copy_all_fits(double lane_bytes, uint64_t ram_avail, double ram_marg// ppts with the 256-row rule against 1324 / 1353 / 1363 with a 1024-row slab (+9%), decode 77.7 against 78.1 tok/s, 268 MB a device (1% of// the card). That branch asks for min(chunk, 1024) rows whatever the spare room (the spare of such a box is negative: its room is all// experts), and never less than the base; the automatic growth on top of it still applies where the room is there. A unified-memory GPU-// (the caller passes env 0 there) and DS4_ATTN_SLAB=0 never take either branch.+// (the caller passes env 0 there) and DS4_ATTN_SLAB=0 never take either branch. The caller passes 0 for `expert_share` in a build that is not+// HIP: that branch is the 4 x R9700 measurement.inline int attn_slab_rows(int env, int chunk, int base, double spare_bytes, double bytes_per_row, double expert_share = 0.0) {auto clampn = [&](int a) { return std::max(64, std::min(chunk, a) / 64 * 64); };if (env == 0) return base;@@ -107,11 +108,13 @@ inline int attn_slab_rows(int env, int chunk, int base, double spare_bytes, doub// summed room, 64 MB a GB, 400..1200 MB. Measured on Strix Halo only (400 -> 1200: the queue 3,200 -> 8,087 experts, 4K prompt 27.3// -> 21.3 s, decode unchanged); a discrete card in this band is not measured and takes the same, VRAM-bounded, rule.// A unified-memory GPU never takes the small ring (its datapoint is the 1200 MB one): it keeps the 16 MB "fit" rule at 1/1.3.-inline int staging_ring_mb(double share, double room_bytes, int n_dev, bool integrated) {+// `hip`: the 0.5..0.82 band and the 0.82 threshold are the 4 x R9700 measurements, so only a HIP build takes them; any other build keeps main's+// rule there (16 MB from 1/1.3, the streamed ring below it) until a CUDA multi-card rig is measured.+inline int staging_ring_mb(double share, double room_bytes, int n_dev, bool integrated, bool hip) {const double GB = 1e9;const double per_dev = n_dev > 0 ? room_bytes / (double) n_dev : 0.0;const double streamed = std::min(1200.0, std::max(400.0, room_bytes / GB * 64.0));- if (integrated) return share >= 1.0 / 1.3 ? 16 : (int) streamed;+ if (integrated || !hip) return share >= 1.0 / 1.3 ? 16 : (int) streamed;if (share >= 0.82) return 16;if (share >= 0.5) return (int) std::min(512.0, std::max(128.0, per_dev / GB * 8.0));return (int) streamed;diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 0286e7165..d35398e21 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -78,6 +78,14 @@ namespace gg = strata::ds4::gguf;namespace {+// The DeepSeek engine's share-based defaults (small chunk, whole-chunk slab, small ring, pinned copy) are the 4 x R9700 measurements: only a HIP+// build takes them.+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)+constexpr bool kHipBuild = true;+#else+constexpr bool kHipBuild = false;+#endif+double now_ms() {return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();}@@ -2032,7 +2040,7 @@ struct Model::Impl {// most (4 x R9700: 8192 214, 1024 370) the small one wins; with the experts copied into RAM (--mmap-experts off) 4096 beats 8192 by 7-9%// (H100 32 GiB: 2048 502, 4096 538, 8192 495). A GGUF pack always has file_experts set (copied only by --mmap-experts off).const bool own_default = !(par || o.tp > 1); // (a run with tp 1 and no --threads: `strata launch` writes --tp 1 for 4+ AMD cards, --tp 2 for 2-3 and nothing for one card = the engine's tp 2)- const bool multi_tp = !own_default && !o.threads && devs.size() >= 2; // (the 4-card default-tp run the 1024 chunk was also measured on; an explicit --prefill-chunk below wins over the rule)+ const bool multi_tp = kHipBuild && !own_default && !o.threads && devs.size() >= 2; // (the 4-card default-tp run the 1024 chunk was also measured on; an explicit --prefill-chunk below wins over the rule)const bool gguf_copy = pk.gguf && (o.mmap_experts == 0 || (store_gb() <= 0.0 && auto_copy_all));uint64_t vmin_bytes = ~0ull;for (const Dev& dv : devs) vmin_bytes = std::min<uint64_t>(vmin_bytes, (uint64_t) dv.total);@@ -2975,7 +2983,7 @@ struct Model::Impl {cudaDeviceProp ip{};if (cudaGetDeviceProperties(&ip, d.id) == cudaSuccess && ip.integrated != 0) unified = true;}- const size_t mb = (size_t) strata::ds4::staging_ring_mb(exp_b > 0 ? share : 0.0, room, (int) devs.size(), unified);+ const size_t mb = (size_t) strata::ds4::staging_ring_mb(exp_b > 0 ? share : 0.0, room, (int) devs.size(), unified, kHipBuild);if (o.verbose)std::fprintf(stderr, "ds4: staging ring: experts %.1f GB, VRAM room %.1f GB, dense %.1f GB -> %d MB\n", exp_b / 1e9,room / 1e9, other_b / 1e9, (int) mb);@@ -3035,11 +3043,11 @@ struct Model::Impl {const int base = c.A;// one rule (ds4_experts_rule.hpp: attn_slab_rows): the chunk's whole slab (<= 1024 rows) where the cards hold half the// experts or more (4 x R9700, +9% prompt), up to 2048 rows where the VRAM is spare, the base otherwise- c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr, share);+ c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr, kHipBuild ? share : 0.0);if (env < 0 && c.A > base) {slab_auto_grew_ = true;if (di == 0 && o.verbose) {- if (share >= 0.5)+ if (kHipBuild && share >= 0.5)std::fprintf(stderr, "ds4: attention slab %d rows (automatic: the cards hold %.0f%% of the experts; DS4_ATTN_SLAB=0 keeps %d)\n",c.A, share * 100.0, base);else@@ -4591,10 +4599,10 @@ struct Model::Impl {// DS4_COPY_PIN_RO=1: cudaHostRegisterReadOnly (the GPU only reads it).// 0.2.7: DS4_COPY_PIN unset = ON when EITHER condition measured reliable holds: (a) the whole arena is on explicit 2 MB pages (a pin of a// THP arena registers 4 KB pages and the DMA rate was erratic on the 12 GB cell: 11.9 s one prompt, 30.4 s the next), or (b) the- // cards hold at least half the routed experts (4 x R9700: the pin was reliable in 4/4 runs there, see above). =0 is the off+ // cards hold at least half the routed experts in a HIP build (4 x R9700: the pin was reliable in 4/4 runs there, see above). =0 is the off// switch, =1 pins any arena.const char* cp = std::getenv("DS4_COPY_PIN");- const bool pin_copies = cp != nullptr ? std::atoi(cp) != 0 : ((arena_all_huge || gpu_expert_share() >= 0.5) && !ram_chunks.empty());+ const bool pin_copies = cp != nullptr ? std::atoi(cp) != 0 : ((arena_all_huge || (kHipBuild && gpu_expert_share() >= 0.5)) && !ram_chunks.empty());if (pin_copies) {const bool ro = std::getenv("DS4_COPY_PIN_RO") != nullptr && std::atoi(std::getenv("DS4_COPY_PIN_RO")) != 0;const double tr = now_ms();diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cppindex fa1691bba..fdaf50fef 100644--- a/tests/dsv41/experts_rule_test.cpp+++ b/tests/dsv41/experts_rule_test.cpp@@ -104,36 +104,42 @@ int main() {CHECK_EQ(attn_slab_rows(512, 1024, 256, 1e18, bpr, 0.77), 512); // DS4_ATTN_SLAB=N: that many rows whatever the shareCHECK_EQ(attn_slab_rows(-1, 1024, 1024, -5.0e9, bpr, 0.77), 1024); // a unified-memory base (32 MB of q = 1024 rows) is kept}- // the staging ring (MB): staging_ring_mb(share, room_bytes, n_dev, integrated)+ // the staging ring (MB): staging_ring_mb(share, room_bytes, n_dev, integrated, hip){const double GB = 1e9;// the cards hold the experts: 16 MB, discrete (4 x R9700 at 0.86 and a card that holds everything)- CHECK_EQ(staging_ring_mb(0.86, 4 * 30 * GB, 4, false), 16);- CHECK_EQ(staging_ring_mb(1.00, 70 * GB, 1, false), 16);- CHECK_EQ(staging_ring_mb(0.82, 4 * 30 * GB, 4, false), 16);+ CHECK_EQ(staging_ring_mb(0.86, 4 * 30 * GB, 4, false, true), 16);+ CHECK_EQ(staging_ring_mb(1.00, 70 * GB, 1, false, true), 16);+ CHECK_EQ(staging_ring_mb(0.82, 4 * 30 * GB, 4, false, true), 16);// they hold most of them (4 x R9700, share 0.72-0.77, 31 GB of room a device): 8 MB a GB of one device's room = 248 MB- CHECK_EQ(staging_ring_mb(0.77, 4 * 31 * GB, 4, false), 248);- CHECK_EQ(staging_ring_mb(0.72, 4 * 31 * GB, 4, false), 248);- CHECK_EQ(staging_ring_mb(0.81, 4 * 31 * GB, 4, false), 248);- CHECK_EQ(staging_ring_mb(0.77, 4 * 10 * GB, 4, false), 128); // small devices: at least 128- CHECK_EQ(staging_ring_mb(0.60, 3 * 90 * GB, 3, false), 512); // big devices: at most 512- CHECK_EQ(staging_ring_mb(0.50, 2 * 31 * GB, 2, false), 248); // the per-device room, not the summed one (1 card: 31 GB -> 248 too)- CHECK_EQ(staging_ring_mb(0.60, 31 * GB, 1, false), 248);+ CHECK_EQ(staging_ring_mb(0.77, 4 * 31 * GB, 4, false, true), 248);+ CHECK_EQ(staging_ring_mb(0.72, 4 * 31 * GB, 4, false, true), 248);+ CHECK_EQ(staging_ring_mb(0.81, 4 * 31 * GB, 4, false, true), 248);+ CHECK_EQ(staging_ring_mb(0.77, 4 * 10 * GB, 4, false, true), 128); // small devices: at least 128+ CHECK_EQ(staging_ring_mb(0.60, 3 * 90 * GB, 3, false, true), 512); // big devices: at most 512+ CHECK_EQ(staging_ring_mb(0.50, 2 * 31 * GB, 2, false, true), 248); // the per-device room, not the summed one (1 card: 31 GB -> 248 too)+ CHECK_EQ(staging_ring_mb(0.60, 31 * GB, 1, false, true), 248);// they hold under half (discrete): 64 MB a GB of the summed room, 400..1200 (streamed; not measured on a discrete card)- CHECK_EQ(staging_ring_mb(0.19, 31 * GB, 1, false), 1200);- CHECK_EQ(staging_ring_mb(0.06, 10 * GB, 1, false), 640); // a 12 GB card- CHECK_EQ(staging_ring_mb(0.06, 4 * GB, 1, false), 400); // a small pool still gets 400- CHECK_EQ(staging_ring_mb(0.40, 2 * 31 * GB, 2, false), 1200); // capped- CHECK_EQ(staging_ring_mb(0.49, 4 * 31 * GB, 4, false), 1200);+ CHECK_EQ(staging_ring_mb(0.19, 31 * GB, 1, false, true), 1200);+ CHECK_EQ(staging_ring_mb(0.06, 10 * GB, 1, false, true), 640); // a 12 GB card+ CHECK_EQ(staging_ring_mb(0.06, 4 * GB, 1, false, true), 400); // a small pool still gets 400+ CHECK_EQ(staging_ring_mb(0.40, 2 * 31 * GB, 2, false, true), 1200); // capped+ CHECK_EQ(staging_ring_mb(0.49, 4 * 31 * GB, 4, false, true), 1200);// a unified-memory GPU (Strix Halo): never the small ring; the streamed 400..1200 MB under 1/1.3, 16 above- CHECK_EQ(staging_ring_mb(0.60, 40 * GB, 1, true), 1200); // share 0.6 would be the small ring on a discrete card- CHECK_EQ(staging_ring_mb(0.30, 18.75 * GB, 1, true), 1200);- CHECK_EQ(staging_ring_mb(0.30, 12 * GB, 1, true), 768);- CHECK_EQ(staging_ring_mb(0.30, 5 * GB, 1, true), 400);- CHECK_EQ(staging_ring_mb(0.77, 40 * GB, 1, true), 16); // main's "fit" threshold, 1/1.3 = 0.769- CHECK_EQ(staging_ring_mb(0.76, 40 * GB, 1, true), 1200);+ CHECK_EQ(staging_ring_mb(0.60, 40 * GB, 1, true, true), 1200); // share 0.6 would be the small ring on a discrete card+ CHECK_EQ(staging_ring_mb(0.30, 18.75 * GB, 1, true, true), 1200);+ CHECK_EQ(staging_ring_mb(0.30, 12 * GB, 1, true, true), 768);+ CHECK_EQ(staging_ring_mb(0.30, 5 * GB, 1, true, true), 400);+ CHECK_EQ(staging_ring_mb(0.77, 40 * GB, 1, true, true), 16); // main's "fit" threshold, 1/1.3 = 0.769+ CHECK_EQ(staging_ring_mb(0.76, 40 * GB, 1, true, true), 1200);// nothing known: no devices- CHECK_EQ(staging_ring_mb(0.0, 0.0, 0, false), 400);+ CHECK_EQ(staging_ring_mb(0.0, 0.0, 0, false, true), 400);+ // a build that is not HIP keeps main's rule everywhere: 16 MB from 1/1.3, the streamed ring below it (no small band, no 0.82)+ CHECK_EQ(staging_ring_mb(0.77, 4 * 31 * GB, 4, false, false), 16);+ CHECK_EQ(staging_ring_mb(0.60, 4 * 31 * GB, 4, false, false), 1200);+ CHECK_EQ(staging_ring_mb(0.80, 4 * 31 * GB, 4, false, false), 16);+ CHECK_EQ(staging_ring_mb(0.06, 10 * GB, 1, false, false), 640);+ CHECK_EQ(staging_ring_mb(0.30, 12 * GB, 1, true, false), 768);}// prompt chunk: prefill_chunk_default(own_default, gpu_share, gguf_copy, file_experts, hip, min_vram, big=8192, multi_tp=false). What `strata launch` runs on one card// passes neither --tp nor --threads, but the engine's default tp is 2, so own_default is false there; a launch with several cards passes --tp 1.