Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
DeepSeek engine: the auto RAM store no longer reads a discrete GPU's pinned buffers as unified memory (#4147)
Niko1221ea73eab44258
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hppindex 79e9c4b9b..fecba633c 100644--- a/src/dsv41/ds4_experts_rule.hpp+++ b/src/dsv41/ds4_experts_rule.hpp@@ -89,6 +89,16 @@ inline bool copy_all_fits(double lane_bytes, uint64_t ram_avail, double ram_margreturn lane_bytes + ram_margin <= (double) ram_avail && lane_bytes <= commit_room;}+// Whether the GPU's allocations come out of the same RAM the auto store would use (Strix Halo, DGX Spark's GB10): only a device the driver reports as+// integrated, AND the free RAM fell by `uma_drop_gb` or more between the first store estimate (before the weights went to the GPU) and now.+// A discrete card (2 x RTX 3090, 2 x A6000) never counts, however much RAM the pinned staging buffers, the page cache and the other programs took+// meanwhile: v0.2.5..0.2.7 read every drop of 4 GB as unified memory and shrank a 256K context's store by 27 GB (issue #4147, 48 -> 17.8 GB).+// avail_before / avail_now: free RAM in bytes (~0: unknown -> no).+inline bool store_gpu_shares_ram(bool any_integrated_device, uint64_t avail_before, uint64_t avail_now, double uma_drop_gb = 4.0) {+ if (!any_integrated_device || avail_before == ~uint64_t{0} || avail_now == ~uint64_t{0}) return false;+ return avail_before > avail_now && (double) (avail_before - avail_now) / 1e9 >= uma_drop_gb;+}+// The chunked prompt's attention slab (rows a pass: the dense projections q_b / wo_a / wo_b are dequantized and multiplied once a slab, so// a 256-row slab dequantizes them 32 times for an 8K prompt). `base`: the 0.2.6 rule (8 MB of rows / heads width, 256 on this model).// DS4_ATTN_SLAB=N (`env` >= 0): 0 gives the 0.2.6 rule back, N > 0 asks for that many rows (64..chunk, a multiple of 64). Unset (`env` < 0):diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 7ffa7cc4f..57dfd860b 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -5205,7 +5205,13 @@ struct Model::Impl {if (avail == ~uint64_t{0}) return gb;// (the prompt buffers and the context's caches are touched after this point and, on such a machine, take RAM too: measured on// Strix Halo about 0.11 MB a token of context beyond 16384, a 40960 context ending 2.5 GB lower than a 16384 one)- const bool uma = avail0_ != ~uint64_t{0} && avail0_ > avail && (double) (avail0_ - avail) / 1e9 >= 4.0;+ bool any_integrated = false; // (only a device the driver calls integrated can share the RAM: a discrete card's pinned buffers also lower the free RAM, #4147)+ for (const Dev& d : devs) {+ int v = 0;+ if (cudaDeviceGetAttribute(&v, cudaDevAttrIntegrated, d.id) == cudaSuccess) any_integrated = any_integrated || v != 0;+ else (void) cudaGetLastError();+ }+ const bool uma = strata::ds4::store_gpu_shares_ram(any_integrated, avail0_, avail);const double ctx_gb = uma ? 1.1e-4 * std::max(0.0, (double) o.max_context - 16384.0) : 0.0;const double room = (double) avail / 1e9 - 12.0 - ctx_gb;const double now = room >= 4.0 ? std::min(store_cap_gb(), room) : 0.0;diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cppindex 895df7f02..8277b85d6 100644--- a/tests/dsv41/experts_rule_test.cpp+++ b/tests/dsv41/experts_rule_test.cpp@@ -10,6 +10,7 @@ using strata::ds4::experts_test_override;using strata::ds4::copy_all_fits;using strata::ds4::attn_slab_rows;using strata::ds4::staging_ring_mb;+using strata::ds4::store_gpu_shares_ram;static int fails = 0;#define CHECK_EQ(a, b) \@@ -307,6 +308,19 @@ int main() {CHECK_EQ(dense_pfgemm_default(1, true), 1);CHECK_EQ(dense_pfgemm_default(0, false), 0);}+ // #4147: the auto store's "the GPU shares the RAM" decision. A discrete 2 x RTX 3090 whose pinned buffers took 7.5 GB of the free RAM is NOT+ // unified memory (the 256K store stayed 48 GB); a Strix Halo / GB10 (integrated) whose weights took RAM is, as before.+ {+ const uint64_t G = 1000000000ull;+ CHECK_EQ(store_gpu_shares_ram(false, 62 * G, 54 * G), 0); // discrete, free RAM fell 8 GB: no+ CHECK_EQ(store_gpu_shares_ram(false, 62 * G, 20 * G), 0); // discrete, even a 42 GB drop: no+ CHECK_EQ(store_gpu_shares_ram(true, 62 * G, 54 * G), 1); // integrated, 8 GB drop: yes (unchanged)+ CHECK_EQ(store_gpu_shares_ram(true, 62 * G, 58 * G), 1); // exactly the 4 GB threshold+ CHECK_EQ(store_gpu_shares_ram(true, 62 * G, 58 * G + 1), 0); // just under it+ CHECK_EQ(store_gpu_shares_ram(true, 54 * G, 62 * G), 0); // RAM grew: no+ CHECK_EQ(store_gpu_shares_ram(true, ~uint64_t{0}, 40 * G), 0); // the first estimate unknown+ CHECK_EQ(store_gpu_shares_ram(true, 62 * G, ~uint64_t{0}), 0);+ }if (fails == 0) std::printf("experts rule: all checks passed\n");return fails == 0 ? 0 : 1;}