Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek: the 4 x R9700 share-based defaults (small chunk for multi-card tp runs, whole-chunk slab, small staging ring, pinned copy at share >= 0.5) apply to HIP builds only; the staging ring takes a hip flag; tests for both branches

Niko1221484c6893375c
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hpp
index 50e23d98f..746822ea7 100644
--- a/src/dsv41/ds4_experts_rule.hpp
+++ b/src/dsv41/ds4_experts_rule.hpp
@@ -47,7 +47,7 @@ inline int prefill_chunk_default(bool own_default, double gpu_share, bool gguf_c
// `strata launch` writes for 4 or more AMD cards), share 0.86: 1024 370 against 214 tok/s at 8192; and with the engine's own default tp
// (2, pairs of cards, nothing written), where the rule used to fall through to 2048/8192: 2.25 bpw, --dspark auto, ids 5705e1164a, 8K
// warm prompts 1024 -> 1247 ppts against 690 at 8192, decode unchanged. So a multi-card tp run takes it too. One card with the default
- // tp, and `--threads on` (it stalled at the first 8K prompt in the 4-card runs), keep their old defaults: not measured with 1024.
+ // tp, `--threads on` (it stalled at the first 8K prompt in the 4-card runs) and a build that is not HIP keep their old defaults: not measured with 1024.
// An explicit --prefill-chunk N wins over all of this (the call site takes the rule only when none was given).
if ((own_default || multi_tp) && gpu_share >= 0.5) return 1024;
if (gguf_copy && !hip && min_vram_bytes >= kChunk4096MinVram) return 4096;
@@ -81,7 +81,8 @@ inline bool copy_all_fits(double lane_bytes, uint64_t ram_avail, double ram_marg
// ppts with the 256-row rule against 1324 / 1353 / 1363 with a 1024-row slab (+9%), decode 77.7 against 78.1 tok/s, 268 MB a device (1% of
// the card). That branch asks for min(chunk, 1024) rows whatever the spare room (the spare of such a box is negative: its room is all
// experts), and never less than the base; the automatic growth on top of it still applies where the room is there. A unified-memory GPU
-// (the caller passes env 0 there) and DS4_ATTN_SLAB=0 never take either branch.
+// (the caller passes env 0 there) and DS4_ATTN_SLAB=0 never take either branch. The caller passes 0 for `expert_share` in a build that is not
+// HIP: that branch is the 4 x R9700 measurement.
inline int attn_slab_rows(int env, int chunk, int base, double spare_bytes, double bytes_per_row, double expert_share = 0.0) {
auto clampn = [&](int a) { return std::max(64, std::min(chunk, a) / 64 * 64); };
if (env == 0) return base;
@@ -107,11 +108,13 @@ inline int attn_slab_rows(int env, int chunk, int base, double spare_bytes, doub
// summed room, 64 MB a GB, 400..1200 MB. Measured on Strix Halo only (400 -> 1200: the queue 3,200 -> 8,087 experts, 4K prompt 27.3
// -> 21.3 s, decode unchanged); a discrete card in this band is not measured and takes the same, VRAM-bounded, rule.
// A unified-memory GPU never takes the small ring (its datapoint is the 1200 MB one): it keeps the 16 MB "fit" rule at 1/1.3.
-inline int staging_ring_mb(double share, double room_bytes, int n_dev, bool integrated) {
+// `hip`: the 0.5..0.82 band and the 0.82 threshold are the 4 x R9700 measurements, so only a HIP build takes them; any other build keeps main's
+// rule there (16 MB from 1/1.3, the streamed ring below it) until a CUDA multi-card rig is measured.
+inline int staging_ring_mb(double share, double room_bytes, int n_dev, bool integrated, bool hip) {
const double GB = 1e9;
const double per_dev = n_dev > 0 ? room_bytes / (double) n_dev : 0.0;
const double streamed = std::min(1200.0, std::max(400.0, room_bytes / GB * 64.0));
- if (integrated) return share >= 1.0 / 1.3 ? 16 : (int) streamed;
+ if (integrated || !hip) return share >= 1.0 / 1.3 ? 16 : (int) streamed;
if (share >= 0.82) return 16;
if (share >= 0.5) return (int) std::min(512.0, std::max(128.0, per_dev / GB * 8.0));
return (int) streamed;
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 0286e7165..d35398e21 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -78,6 +78,14 @@ namespace gg = strata::ds4::gguf;
namespace {
+// The DeepSeek engine's share-based defaults (small chunk, whole-chunk slab, small ring, pinned copy) are the 4 x R9700 measurements: only a HIP
+// build takes them.
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+constexpr bool kHipBuild = true;
+#else
+constexpr bool kHipBuild = false;
+#endif
+
double now_ms() {
return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now().time_since_epoch()).count();
}
@@ -2032,7 +2040,7 @@ struct Model::Impl {
// most (4 x R9700: 8192 214, 1024 370) the small one wins; with the experts copied into RAM (--mmap-experts off) 4096 beats 8192 by 7-9%
// (H100 32 GiB: 2048 502, 4096 538, 8192 495). A GGUF pack always has file_experts set (copied only by --mmap-experts off).
const bool own_default = !(par || o.tp > 1); // (a run with tp 1 and no --threads: `strata launch` writes --tp 1 for 4+ AMD cards, --tp 2 for 2-3 and nothing for one card = the engine's tp 2)
- const bool multi_tp = !own_default && !o.threads && devs.size() >= 2; // (the 4-card default-tp run the 1024 chunk was also measured on; an explicit --prefill-chunk below wins over the rule)
+ const bool multi_tp = kHipBuild && !own_default && !o.threads && devs.size() >= 2; // (the 4-card default-tp run the 1024 chunk was also measured on; an explicit --prefill-chunk below wins over the rule)
const bool gguf_copy = pk.gguf && (o.mmap_experts == 0 || (store_gb() <= 0.0 && auto_copy_all));
uint64_t vmin_bytes = ~0ull;
for (const Dev& dv : devs) vmin_bytes = std::min<uint64_t>(vmin_bytes, (uint64_t) dv.total);
@@ -2975,7 +2983,7 @@ struct Model::Impl {
cudaDeviceProp ip{};
if (cudaGetDeviceProperties(&ip, d.id) == cudaSuccess && ip.integrated != 0) unified = true;
}
- const size_t mb = (size_t) strata::ds4::staging_ring_mb(exp_b > 0 ? share : 0.0, room, (int) devs.size(), unified);
+ const size_t mb = (size_t) strata::ds4::staging_ring_mb(exp_b > 0 ? share : 0.0, room, (int) devs.size(), unified, kHipBuild);
if (o.verbose)
std::fprintf(stderr, "ds4: staging ring: experts %.1f GB, VRAM room %.1f GB, dense %.1f GB -> %d MB\n", exp_b / 1e9,
room / 1e9, other_b / 1e9, (int) mb);
@@ -3035,11 +3043,11 @@ struct Model::Impl {
const int base = c.A;
// one rule (ds4_experts_rule.hpp: attn_slab_rows): the chunk's whole slab (<= 1024 rows) where the cards hold half the
// experts or more (4 x R9700, +9% prompt), up to 2048 rows where the VRAM is spare, the base otherwise
- c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr, share);
+ c.A = strata::ds4::attn_slab_rows(env, chunk, base, spare, bpr, kHipBuild ? share : 0.0);
if (env < 0 && c.A > base) {
slab_auto_grew_ = true;
if (di == 0 && o.verbose) {
- if (share >= 0.5)
+ if (kHipBuild && share >= 0.5)
std::fprintf(stderr, "ds4: attention slab %d rows (automatic: the cards hold %.0f%% of the experts; DS4_ATTN_SLAB=0 keeps %d)\n",
c.A, share * 100.0, base);
else
@@ -4591,10 +4599,10 @@ struct Model::Impl {
// DS4_COPY_PIN_RO=1: cudaHostRegisterReadOnly (the GPU only reads it).
// 0.2.7: DS4_COPY_PIN unset = ON when EITHER condition measured reliable holds: (a) the whole arena is on explicit 2 MB pages (a pin of a
// THP arena registers 4 KB pages and the DMA rate was erratic on the 12 GB cell: 11.9 s one prompt, 30.4 s the next), or (b) the
- // cards hold at least half the routed experts (4 x R9700: the pin was reliable in 4/4 runs there, see above). =0 is the off
+ // cards hold at least half the routed experts in a HIP build (4 x R9700: the pin was reliable in 4/4 runs there, see above). =0 is the off
// switch, =1 pins any arena.
const char* cp = std::getenv("DS4_COPY_PIN");
- const bool pin_copies = cp != nullptr ? std::atoi(cp) != 0 : ((arena_all_huge || gpu_expert_share() >= 0.5) && !ram_chunks.empty());
+ const bool pin_copies = cp != nullptr ? std::atoi(cp) != 0 : ((arena_all_huge || (kHipBuild && gpu_expert_share() >= 0.5)) && !ram_chunks.empty());
if (pin_copies) {
const bool ro = std::getenv("DS4_COPY_PIN_RO") != nullptr && std::atoi(std::getenv("DS4_COPY_PIN_RO")) != 0;
const double tr = now_ms();
diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cpp
index fa1691bba..fdaf50fef 100644
--- a/tests/dsv41/experts_rule_test.cpp
+++ b/tests/dsv41/experts_rule_test.cpp
@@ -104,36 +104,42 @@ int main() {
CHECK_EQ(attn_slab_rows(512, 1024, 256, 1e18, bpr, 0.77), 512); // DS4_ATTN_SLAB=N: that many rows whatever the share
CHECK_EQ(attn_slab_rows(-1, 1024, 1024, -5.0e9, bpr, 0.77), 1024); // a unified-memory base (32 MB of q = 1024 rows) is kept
}
- // the staging ring (MB): staging_ring_mb(share, room_bytes, n_dev, integrated)
+ // the staging ring (MB): staging_ring_mb(share, room_bytes, n_dev, integrated, hip)
{
const double GB = 1e9;
// the cards hold the experts: 16 MB, discrete (4 x R9700 at 0.86 and a card that holds everything)
- CHECK_EQ(staging_ring_mb(0.86, 4 * 30 * GB, 4, false), 16);
- CHECK_EQ(staging_ring_mb(1.00, 70 * GB, 1, false), 16);
- CHECK_EQ(staging_ring_mb(0.82, 4 * 30 * GB, 4, false), 16);
+ CHECK_EQ(staging_ring_mb(0.86, 4 * 30 * GB, 4, false, true), 16);
+ CHECK_EQ(staging_ring_mb(1.00, 70 * GB, 1, false, true), 16);
+ CHECK_EQ(staging_ring_mb(0.82, 4 * 30 * GB, 4, false, true), 16);
// they hold most of them (4 x R9700, share 0.72-0.77, 31 GB of room a device): 8 MB a GB of one device's room = 248 MB
- CHECK_EQ(staging_ring_mb(0.77, 4 * 31 * GB, 4, false), 248);
- CHECK_EQ(staging_ring_mb(0.72, 4 * 31 * GB, 4, false), 248);
- CHECK_EQ(staging_ring_mb(0.81, 4 * 31 * GB, 4, false), 248);
- CHECK_EQ(staging_ring_mb(0.77, 4 * 10 * GB, 4, false), 128); // small devices: at least 128
- CHECK_EQ(staging_ring_mb(0.60, 3 * 90 * GB, 3, false), 512); // big devices: at most 512
- CHECK_EQ(staging_ring_mb(0.50, 2 * 31 * GB, 2, false), 248); // the per-device room, not the summed one (1 card: 31 GB -> 248 too)
- CHECK_EQ(staging_ring_mb(0.60, 31 * GB, 1, false), 248);
+ CHECK_EQ(staging_ring_mb(0.77, 4 * 31 * GB, 4, false, true), 248);
+ CHECK_EQ(staging_ring_mb(0.72, 4 * 31 * GB, 4, false, true), 248);
+ CHECK_EQ(staging_ring_mb(0.81, 4 * 31 * GB, 4, false, true), 248);
+ CHECK_EQ(staging_ring_mb(0.77, 4 * 10 * GB, 4, false, true), 128); // small devices: at least 128
+ CHECK_EQ(staging_ring_mb(0.60, 3 * 90 * GB, 3, false, true), 512); // big devices: at most 512
+ CHECK_EQ(staging_ring_mb(0.50, 2 * 31 * GB, 2, false, true), 248); // the per-device room, not the summed one (1 card: 31 GB -> 248 too)
+ CHECK_EQ(staging_ring_mb(0.60, 31 * GB, 1, false, true), 248);
// they hold under half (discrete): 64 MB a GB of the summed room, 400..1200 (streamed; not measured on a discrete card)
- CHECK_EQ(staging_ring_mb(0.19, 31 * GB, 1, false), 1200);
- CHECK_EQ(staging_ring_mb(0.06, 10 * GB, 1, false), 640); // a 12 GB card
- CHECK_EQ(staging_ring_mb(0.06, 4 * GB, 1, false), 400); // a small pool still gets 400
- CHECK_EQ(staging_ring_mb(0.40, 2 * 31 * GB, 2, false), 1200); // capped
- CHECK_EQ(staging_ring_mb(0.49, 4 * 31 * GB, 4, false), 1200);
+ CHECK_EQ(staging_ring_mb(0.19, 31 * GB, 1, false, true), 1200);
+ CHECK_EQ(staging_ring_mb(0.06, 10 * GB, 1, false, true), 640); // a 12 GB card
+ CHECK_EQ(staging_ring_mb(0.06, 4 * GB, 1, false, true), 400); // a small pool still gets 400
+ CHECK_EQ(staging_ring_mb(0.40, 2 * 31 * GB, 2, false, true), 1200); // capped
+ CHECK_EQ(staging_ring_mb(0.49, 4 * 31 * GB, 4, false, true), 1200);
// a unified-memory GPU (Strix Halo): never the small ring; the streamed 400..1200 MB under 1/1.3, 16 above
- CHECK_EQ(staging_ring_mb(0.60, 40 * GB, 1, true), 1200); // share 0.6 would be the small ring on a discrete card
- CHECK_EQ(staging_ring_mb(0.30, 18.75 * GB, 1, true), 1200);
- CHECK_EQ(staging_ring_mb(0.30, 12 * GB, 1, true), 768);
- CHECK_EQ(staging_ring_mb(0.30, 5 * GB, 1, true), 400);
- CHECK_EQ(staging_ring_mb(0.77, 40 * GB, 1, true), 16); // main's "fit" threshold, 1/1.3 = 0.769
- CHECK_EQ(staging_ring_mb(0.76, 40 * GB, 1, true), 1200);
+ CHECK_EQ(staging_ring_mb(0.60, 40 * GB, 1, true, true), 1200); // share 0.6 would be the small ring on a discrete card
+ CHECK_EQ(staging_ring_mb(0.30, 18.75 * GB, 1, true, true), 1200);
+ CHECK_EQ(staging_ring_mb(0.30, 12 * GB, 1, true, true), 768);
+ CHECK_EQ(staging_ring_mb(0.30, 5 * GB, 1, true, true), 400);
+ CHECK_EQ(staging_ring_mb(0.77, 40 * GB, 1, true, true), 16); // main's "fit" threshold, 1/1.3 = 0.769
+ CHECK_EQ(staging_ring_mb(0.76, 40 * GB, 1, true, true), 1200);
// nothing known: no devices
- CHECK_EQ(staging_ring_mb(0.0, 0.0, 0, false), 400);
+ CHECK_EQ(staging_ring_mb(0.0, 0.0, 0, false, true), 400);
+ // a build that is not HIP keeps main's rule everywhere: 16 MB from 1/1.3, the streamed ring below it (no small band, no 0.82)
+ CHECK_EQ(staging_ring_mb(0.77, 4 * 31 * GB, 4, false, false), 16);
+ CHECK_EQ(staging_ring_mb(0.60, 4 * 31 * GB, 4, false, false), 1200);
+ CHECK_EQ(staging_ring_mb(0.80, 4 * 31 * GB, 4, false, false), 16);
+ CHECK_EQ(staging_ring_mb(0.06, 10 * GB, 1, false, false), 640);
+ CHECK_EQ(staging_ring_mb(0.30, 12 * GB, 1, true, false), 768);
}
// prompt chunk: prefill_chunk_default(own_default, gpu_share, gguf_copy, file_experts, hip, min_vram, big=8192, multi_tp=false). What `strata launch` runs on one card
// passes neither --tp nor --threads, but the engine's default tp is 2, so own_default is false there; a launch with several cards passes --tp 1.