Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

expert cache: STRATA_VRAM_FREE=device reads the whole card's free VRAM through NVML, Windows + NVIDIA, opt-in (#1911)

Xiaomi20083100ca05ed1a
diff --git a/docs/DETAILS.md b/docs/DETAILS.md
index 15f377c71..a53ede74f 100644
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@ -1287,6 +1287,7 @@ with `--batch 2` at 262,144 context saw 538 MiB at the free line become 395 MiB
| `STRATA_SPEC_COUPLED` / `STRATA_SPEC_PROB` (0.1.40.2) | RTX 3060 IQ3_XXS, 200-token answers, 10 pairs | Median tok/s at temperature 0.3 / 0.7 / 1.0: default 42.4 / 43.2 / 43.3, coupled 40.6 / 42.4 / 43.3, rejection sampling 41.0 / 42.6 / 42.6 |
| `STRATA_STAGER_SLEEP=1` | Ryzen 9 7940HS laptop, RTX 4070, IQ3_S, 64K (issue #1101) | Prompts of 5,914 and 22,305 tokens read 5-6% slower; whole-machine CPU use 77-90% -> 21-25%. An RTX 5070 read an 8K Q2_0 prompt 1.2% faster with spinning |
| Stager threads for a GGUF read in place | Big-RAM box, 124K-token Q8 prompt | 57 -> 350-410 tok/s with the 4-thread RAM profile instead of the 32-thread SSD profile (#1353) |
+| `STRATA_VRAM_FREE=device` (Windows, NVIDIA, #1911) | RTX 5090 (WDDM), Core i7-13700K, 64 GB, 128K context, image encoder loaded, `--expert-cache auto` | The switch reads the free VRAM of the whole card through NVML: with 6 GiB held by another process CUDA still read 30,994 MiB free where NVML read 24,348. Default starts ended with 336 to 602 MiB free (0.1.42, three starts) and 93 MiB to 10 GiB (0.1.40.3 and 0.1.41, six starts); with the switch and `--vram-reserve-mib 1500` 973 to 1,392 MiB free over seven starts and 1,107 to 1,288 over three on 0.1.42. A run that filled the card to 0 MiB free decoded 80-115 tok/s where the same text ran at 205-230 with 900 MiB or more free. The cache can only get smaller. Not measured: AMD cards and Linux (the switch is compiled for Windows CUDA builds only). |
### Threads, sockets and sessions
diff --git a/include/strata/core/expert_cache.hpp b/include/strata/core/expert_cache.hpp
index 9685aa4d5..3f66f5ca7 100644
--- a/include/strata/core/expert_cache.hpp
+++ b/include/strata/core/expert_cache.hpp
@@ -75,7 +75,9 @@ bool write_expert_profile(const std::string& path, int64_t n_layers, int64_t n_e
/// Free device memory for the expert cache. On a unified memory system - one memory for the CPU and the GPU, such as
/// the DGX Spark (GB10); CUDA reports it as cudaDevAttrIntegrated - cudaMemGetInfo counts only MemFree, not the page cache - which, with the experts read from the GGUF in place, is mostly those
/// files' clean pages and gives way to a device allocation. There the figure is MemAvailable less 6 GiB for the OS
-/// and the engine's host side (STRATA_UMA_HEADROOM_GIB). Elsewhere it is cudaMemGetInfo's.
+/// and the engine's host side (STRATA_UMA_HEADROOM_GIB). Elsewhere it is cudaMemGetInfo's. On Windows with an NVIDIA
+/// card STRATA_VRAM_FREE=device (opt-in) takes the smaller of that and NVML's figure for the whole card: under WDDM
+/// cudaMemGetInfo does not count what other processes hold. It can only make the figure smaller.
size_t device_free_bytes();
/// One piece of an expert blob that lives contiguously in a source's own memory: `n` bytes at `p`, to land `at` bytes into the blob.
diff --git a/src/core/expert_cache.cpp b/src/core/expert_cache.cpp
index bd98ad711..f499378c3 100644
--- a/src/core/expert_cache.cpp
+++ b/src/core/expert_cache.cpp
@@ -22,8 +22,65 @@
#include <utility>
#include <cstring>
+// (not in a SYCL build: its CUDA names are a compatibility layer without the PCI ids NVML is matched by)
+#if defined(_WIN32) && !defined(STRATA_EC_NO_VMM) && !defined(SYCL_LANGUAGE_VERSION)
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#define STRATA_EC_NVML_FREE 1
+#endif
+
namespace strata::core {
+#if defined(STRATA_EC_NVML_FREE)
+namespace {
+
+// STRATA_VRAM_FREE=device (opt-in, Windows + NVIDIA). Under WDDM cudaMemGetInfo answers for THIS process: memory
+// another process holds on the card (the image encoder, a browser, another engine) is not in its free figure.
+// Measured on an RTX 5090, driver 610.88: with 6 GiB written by another process, cudaMemGetInfo in a second process
+// still read 30,994 MiB free where NVML read 24,348. NVML (nvml.dll, installed with every driver) counts the whole
+// card. The library is loaded from the system directory only, at first use; false when it cannot answer.
+bool nvml_device_free(size_t& out) {
+ struct Memory { unsigned long long total, free, used; };
+ typedef int (*InitFn)(void);
+ typedef int (*ByBusFn)(const char*, void**);
+ typedef int (*MemFn)(void*, Memory*);
+ struct Lib {
+ ByBusFn by_bus = nullptr;
+ MemFn mem = nullptr;
+ Lib() {
+ HMODULE h = LoadLibraryExW(L"nvml.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32);
+ if (h == nullptr) return;
+ const auto init = (InitFn) (void*) GetProcAddress(h, "nvmlInit_v2");
+ const auto bus = (ByBusFn) (void*) GetProcAddress(h, "nvmlDeviceGetHandleByPciBusId_v2");
+ const auto m = (MemFn) (void*) GetProcAddress(h, "nvmlDeviceGetMemoryInfo");
+ if (init == nullptr || bus == nullptr || m == nullptr || init() != 0) return;
+ by_bus = bus;
+ mem = m;
+ }
+ };
+ static const Lib lib;
+ if (lib.mem == nullptr) return false;
+ int dev = 0;
+ cudaDeviceProp prop{};
+ if (cudaGetDevice(&dev) != cudaSuccess || cudaGetDeviceProperties(&prop, dev) != cudaSuccess) return false;
+ char bus[32];
+ std::snprintf(bus, sizeof bus, "%08X:%02X:%02X.0", (unsigned) prop.pciDomainID, (unsigned) prop.pciBusID,
+ (unsigned) prop.pciDeviceID);
+ void* handle = nullptr;
+ Memory m{};
+ if (lib.by_bus(bus, &handle) != 0 || lib.mem(handle, &m) != 0) return false;
+ out = (size_t) m.free;
+ return true;
+}
+
+} // namespace
+#endif
+
// See the header. NVIDIA's DGX Spark Porting Guide (section 5.5, "Memory reporting on UMA systems") recommends the same:
// not to rely on cudaMemGetInfo alone but to count the memory the OS can reclaim. Swap is not counted here (unlike
// NVIDIA's reference snippet): an expert cache that pushes the system into swap would be far slower than a smaller one.
@@ -67,6 +124,27 @@ size_t device_free_bytes() {
if (avail > head && avail - head > free_b) free_b = (size_t) (avail - head);
}
}
+#endif
+#if defined(STRATA_EC_NVML_FREE)
+ static const bool device_wide = [] {
+ const char* v = std::getenv("STRATA_VRAM_FREE");
+ return v != nullptr && std::strcmp(v, "device") == 0;
+ }();
+ if (device_wide) {
+ size_t nvml_b = 0;
+ const bool got = nvml_device_free(nvml_b);
+ static std::atomic<bool> said{false};
+ if (!said.exchange(true)) {
+ if (got)
+ std::fprintf(stderr, "strata generate: STRATA_VRAM_FREE=device: free VRAM is read for the whole card "
+ "(NVML: %lld MiB now; CUDA's figure for this process alone: %lld MiB)\n",
+ (long long) (nvml_b >> 20), (long long) (free_b >> 20));
+ else
+ std::fprintf(stderr, "strata generate: STRATA_VRAM_FREE=device: NVML did not answer (nvml.dll); free "
+ "VRAM stays CUDA's figure for this process\n");
+ }
+ if (got) free_b = std::min(free_b, nvml_b);
+ }
#endif
return free_b;
}
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 5decbae3c..77efc0709 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -11194,6 +11194,8 @@ int main(int argc, char** argv) {
// and a page-in while the verify graph spins on a host flag stalls the request for good
size_t free_b = 0, total_b = 0;
cudaMemGetInfo(&free_b, &total_b);
+ // STRATA_VRAM_FREE=device: the whole card's figure when it is the smaller one (the same value otherwise)
+ free_b = std::min(free_b, strata::core::device_free_bytes());
// below ~256 MiB a later allocation (a first-used window's buffers, the desktop, another program) can make
// the driver page GPU memory, and a verify graph spinning on a host flag then never finishes
const int64_t free_mib = (int64_t) (free_b >> 20);