Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

DeepSeek engine: the auto RAM store no longer reads a discrete GPU's pinned buffers as unified memory (#4147)

Niko1221ea73eab44258
diff --git a/src/dsv41/ds4_experts_rule.hpp b/src/dsv41/ds4_experts_rule.hpp
index 79e9c4b9b..fecba633c 100644
--- a/src/dsv41/ds4_experts_rule.hpp
+++ b/src/dsv41/ds4_experts_rule.hpp
@@ -89,6 +89,16 @@ inline bool copy_all_fits(double lane_bytes, uint64_t ram_avail, double ram_marg
return lane_bytes + ram_margin <= (double) ram_avail && lane_bytes <= commit_room;
}
+// Whether the GPU's allocations come out of the same RAM the auto store would use (Strix Halo, DGX Spark's GB10): only a device the driver reports as
+// integrated, AND the free RAM fell by `uma_drop_gb` or more between the first store estimate (before the weights went to the GPU) and now.
+// A discrete card (2 x RTX 3090, 2 x A6000) never counts, however much RAM the pinned staging buffers, the page cache and the other programs took
+// meanwhile: v0.2.5..0.2.7 read every drop of 4 GB as unified memory and shrank a 256K context's store by 27 GB (issue #4147, 48 -> 17.8 GB).
+// avail_before / avail_now: free RAM in bytes (~0: unknown -> no).
+inline bool store_gpu_shares_ram(bool any_integrated_device, uint64_t avail_before, uint64_t avail_now, double uma_drop_gb = 4.0) {
+ if (!any_integrated_device || avail_before == ~uint64_t{0} || avail_now == ~uint64_t{0}) return false;
+ return avail_before > avail_now && (double) (avail_before - avail_now) / 1e9 >= uma_drop_gb;
+}
+
// The chunked prompt's attention slab (rows a pass: the dense projections q_b / wo_a / wo_b are dequantized and multiplied once a slab, so
// a 256-row slab dequantizes them 32 times for an 8K prompt). `base`: the 0.2.6 rule (8 MB of rows / heads width, 256 on this model).
// DS4_ATTN_SLAB=N (`env` >= 0): 0 gives the 0.2.6 rule back, N > 0 asks for that many rows (64..chunk, a multiple of 64). Unset (`env` < 0):
diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cu
index 7ffa7cc4f..57dfd860b 100644
--- a/src/dsv41/ds4_model.cu
+++ b/src/dsv41/ds4_model.cu
@@ -5205,7 +5205,13 @@ struct Model::Impl {
if (avail == ~uint64_t{0}) return gb;
// (the prompt buffers and the context's caches are touched after this point and, on such a machine, take RAM too: measured on
// Strix Halo about 0.11 MB a token of context beyond 16384, a 40960 context ending 2.5 GB lower than a 16384 one)
- const bool uma = avail0_ != ~uint64_t{0} && avail0_ > avail && (double) (avail0_ - avail) / 1e9 >= 4.0;
+ bool any_integrated = false; // (only a device the driver calls integrated can share the RAM: a discrete card's pinned buffers also lower the free RAM, #4147)
+ for (const Dev& d : devs) {
+ int v = 0;
+ if (cudaDeviceGetAttribute(&v, cudaDevAttrIntegrated, d.id) == cudaSuccess) any_integrated = any_integrated || v != 0;
+ else (void) cudaGetLastError();
+ }
+ const bool uma = strata::ds4::store_gpu_shares_ram(any_integrated, avail0_, avail);
const double ctx_gb = uma ? 1.1e-4 * std::max(0.0, (double) o.max_context - 16384.0) : 0.0;
const double room = (double) avail / 1e9 - 12.0 - ctx_gb;
const double now = room >= 4.0 ? std::min(store_cap_gb(), room) : 0.0;
diff --git a/tests/dsv41/experts_rule_test.cpp b/tests/dsv41/experts_rule_test.cpp
index 895df7f02..8277b85d6 100644
--- a/tests/dsv41/experts_rule_test.cpp
+++ b/tests/dsv41/experts_rule_test.cpp
@@ -10,6 +10,7 @@ using strata::ds4::experts_test_override;
using strata::ds4::copy_all_fits;
using strata::ds4::attn_slab_rows;
using strata::ds4::staging_ring_mb;
+using strata::ds4::store_gpu_shares_ram;
static int fails = 0;
#define CHECK_EQ(a, b) \
@@ -307,6 +308,19 @@ int main() {
CHECK_EQ(dense_pfgemm_default(1, true), 1);
CHECK_EQ(dense_pfgemm_default(0, false), 0);
}
+ // #4147: the auto store's "the GPU shares the RAM" decision. A discrete 2 x RTX 3090 whose pinned buffers took 7.5 GB of the free RAM is NOT
+ // unified memory (the 256K store stayed 48 GB); a Strix Halo / GB10 (integrated) whose weights took RAM is, as before.
+ {
+ const uint64_t G = 1000000000ull;
+ CHECK_EQ(store_gpu_shares_ram(false, 62 * G, 54 * G), 0); // discrete, free RAM fell 8 GB: no
+ CHECK_EQ(store_gpu_shares_ram(false, 62 * G, 20 * G), 0); // discrete, even a 42 GB drop: no
+ CHECK_EQ(store_gpu_shares_ram(true, 62 * G, 54 * G), 1); // integrated, 8 GB drop: yes (unchanged)
+ CHECK_EQ(store_gpu_shares_ram(true, 62 * G, 58 * G), 1); // exactly the 4 GB threshold
+ CHECK_EQ(store_gpu_shares_ram(true, 62 * G, 58 * G + 1), 0); // just under it
+ CHECK_EQ(store_gpu_shares_ram(true, 54 * G, 62 * G), 0); // RAM grew: no
+ CHECK_EQ(store_gpu_shares_ram(true, ~uint64_t{0}, 40 * G), 0); // the first estimate unknown
+ CHECK_EQ(store_gpu_shares_ram(true, 62 * G, ~uint64_t{0}), 0);
+ }
if (fails == 0) std::printf("experts rule: all checks passed\n");
return fails == 0 ? 0 : 1;
}