Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Linux integrated GPU: the auto expert cache counts the RAM once (cache <= RAM available - desktop reserve); the APU warning uses the same reserve

Niko1221f8019539926f
diff --git a/CMakeLists.txt b/CMakeLists.txt
index fa275e8be..1a7691f43 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1413,6 +1413,9 @@ if(STRATA_BUILD_TESTS)
add_executable(arch_defaults_test src/core/arch_defaults_test.cpp src/core/arch_defaults.cpp)
target_include_directories(arch_defaults_test PRIVATE include)
add_test(NAME arch_defaults_test COMMAND arch_defaults_test)
+ add_executable(uma_cache_test src/core/uma_cache_test.cpp)
+ target_include_directories(uma_cache_test PRIVATE include)
+ add_test(NAME uma_cache_test COMMAND uma_cache_test)
add_executable(suffix_drafter_test src/spec/suffix_drafter_test.cpp)
target_link_libraries(suffix_drafter_test PRIVATE strata_spec)
add_test(NAME suffix_drafter_test COMMAND suffix_drafter_test)
diff --git a/include/strata/core/uma_cache.hpp b/include/strata/core/uma_cache.hpp
new file mode 100644
index 000000000..50b929cd7
--- /dev/null
+++ b/include/strata/core/uma_cache.hpp
@@ -0,0 +1,34 @@
+// include/strata/core/uma_cache.hpp - sizing the GPU expert cache on a device whose memory IS the system RAM.
+//
+// On a Strix Halo (or any APU) "free VRAM" is the GPU-addressable pool: the BIOS carve-out plus the GTT, up to nearly all of the RAM when
+// the kernel was told ttm.pages_limit (docs/STRIX_HALO.md recommends 112 GiB of a 128 GiB box). The expert arena (every expert, in RAM) is
+// already counted in what the OS has used, but the cache copies experts INTO that same RAM, so a cache sized from the pool alone can ask for
+// more bytes than the OS has left - the OOM killer then takes the engine, or the desktop. The auto cache is therefore sized from
+// min(what the GPU says is free, the RAM the OS says is available - a reserve for the desktop). Only the AUTO size: --expert-cache N is
+// kept as asked (the engine warns). STRATA_UMA_RESERVE_GIB=N sets the reserve (0 = no reserve: the old sizing).
+#pragma once
+
+#include <cstdint>
+#include <cstdlib>
+
+namespace strata::core {
+
+/// The RAM an integrated GPU's expert cache leaves to the OS and the desktop: STRATA_UMA_RESERVE_GIB when set (0 = none), else the larger of
+/// 8 GiB and 8% of the installed RAM (a desktop session with a browser needs ~6 GiB; 8% is ~10 GiB on 128 GB).
+inline uint64_t uma_desktop_reserve_bytes(uint64_t total_ram, const char* env_gib = std::getenv("STRATA_UMA_RESERVE_GIB")) {
+ constexpr uint64_t GiB = 1ull << 30;
+ if (env_gib != nullptr && env_gib[0] != '\0') {
+ char* end = nullptr;
+ const double v = std::strtod(env_gib, &end);
+ if (end != env_gib && v >= 0.0) return (uint64_t) (v * (double) GiB);
+ }
+ const uint64_t pct = total_ram / 100 * 8;
+ return pct > 8 * GiB ? pct : 8 * GiB;
+}
+
+/// The bytes the auto cache may use: the RAM available now, less the reserve (0 when that is already gone).
+inline uint64_t uma_cache_room_bytes(uint64_t available, uint64_t reserve) {
+ return available > reserve ? available - reserve : 0;
+}
+
+} // namespace strata::core
diff --git a/src/core/uma_cache_test.cpp b/src/core/uma_cache_test.cpp
new file mode 100644
index 000000000..249d4fe8e
--- /dev/null
+++ b/src/core/uma_cache_test.cpp
@@ -0,0 +1,32 @@
+// src/core/uma_cache_test.cpp - the RAM reserve of an integrated GPU's auto expert cache (no GPU needed).
+#include "strata/core/uma_cache.hpp"
+
+#include <cstdio>
+
+static int fails = 0;
+#define CHECK(c) do { if (!(c)) { std::printf("FAIL line %d: %s\n", __LINE__, #c); ++fails; } } while (0)
+
+int main() {
+ using namespace strata::core;
+ constexpr uint64_t GiB = 1ull << 30;
+ // the larger of 8 GiB and 8% of the RAM
+ CHECK(uma_desktop_reserve_bytes(32 * GiB, nullptr) == 8 * GiB);
+ CHECK(uma_desktop_reserve_bytes(64 * GiB, nullptr) == 8 * GiB); // 8% = 5.1 GiB
+ CHECK(uma_desktop_reserve_bytes(128 * GiB, nullptr) > 10 * GiB); // 8% = 10.2 GiB
+ CHECK(uma_desktop_reserve_bytes(128 * GiB, nullptr) < 11 * GiB);
+ CHECK(uma_desktop_reserve_bytes(0, nullptr) == 8 * GiB); // RAM unknown: the floor
+ // the person's own figure, 0 included
+ CHECK(uma_desktop_reserve_bytes(128 * GiB, "0") == 0);
+ CHECK(uma_desktop_reserve_bytes(128 * GiB, "4") == 4 * GiB);
+ CHECK(uma_desktop_reserve_bytes(128 * GiB, "2.5") == 2 * GiB + GiB / 2);
+ CHECK(uma_desktop_reserve_bytes(128 * GiB, "") > 10 * GiB); // empty: not set
+ CHECK(uma_desktop_reserve_bytes(64 * GiB, "x") == 8 * GiB); // not a number: ignored
+ CHECK(uma_desktop_reserve_bytes(64 * GiB, "-3") == 8 * GiB);
+ // the room
+ CHECK(uma_cache_room_bytes(100 * GiB, 10 * GiB) == 90 * GiB);
+ CHECK(uma_cache_room_bytes(10 * GiB, 10 * GiB) == 0);
+ CHECK(uma_cache_room_bytes(5 * GiB, 10 * GiB) == 0);
+ CHECK(uma_cache_room_bytes(5 * GiB, 0) == 5 * GiB);
+ if (fails == 0) std::printf("uma_cache_test: all checks passed\n");
+ return fails == 0 ? 0 : 1;
+}
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 3cef8621c..b317b68ad 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -33,6 +33,7 @@
#include "strata/core/foresight_swap.hpp"
#include "strata/core/pinned.hpp"
#include "strata/core/pin_budget.hpp"
+#include "strata/core/uma_cache.hpp"
#include "strata/core/remote_experts.hpp"
#include "strata/core/on_device.hpp"
#include "strata/core/peer_experts.hpp"
@@ -5931,6 +5932,32 @@ int main(int argc, char** argv) {
(void) cudaGetLastError();
free_b = strata::core::device_free_bytes();
}
+#if defined(STRATA_USE_HIP) && !defined(_WIN32)
+ { // An integrated GPU's "free VRAM" is system RAM (the BIOS carve-out plus the GTT pool, nearly all of the RAM when the kernel was told
+ // ttm.pages_limit), and the cache copies experts into that same RAM on top of the arena: sized from the pool alone it can ask for
+ // more than the OS has left, and the OOM killer takes the engine or the desktop. So the AUTO size counts that RAM once: the
+ // cache gets at most what the OS reports available less a reserve for the desktop (include/strata/core/uma_cache.hpp).
+ // A recommendation: --expert-cache N is kept as asked, STRATA_UMA_RESERVE_GIB=0 gives the old sizing.
+ cudaDeviceProp up{};
+ int ud = 0;
+ strata::core::detail::HostMemory uh{};
+ if (cudaGetDevice(&ud) == cudaSuccess && cudaGetDeviceProperties(&up, ud) == cudaSuccess && up.integrated &&
+ strata::core::detail::host_available_memory(uh)) {
+ const uint64_t reserve = strata::core::uma_desktop_reserve_bytes(strata::platform::total_physical_memory());
+ const uint64_t room = strata::core::uma_cache_room_bytes((uint64_t) uh.available, reserve);
+ if (reserve > 0 && room < (uint64_t) free_b) {
+ std::fprintf(stderr, "strata generate: expert cache auto: this GPU shares system RAM: %.2f GiB free on the GPU, but the OS has "
+ "%.2f GiB available; %.2f GiB stay for the desktop, so the cache is sized from %.2f GiB "
+ "(STRATA_UMA_RESERVE_GIB changes the reserve, --expert-cache N sets the cache)\n",
+ (double) free_b / 1073741824.0, (double) uh.available / 1073741824.0,
+ (double) reserve / 1073741824.0, (double) room / 1073741824.0);
+ free_b = (size_t) room;
+ }
+ } else {
+ (void) cudaGetLastError();
+ }
+ }
+#endif
// Plan v0.3 P5: the batched prompt path's chunk buffers are allocated later, so they are reserved here -
// under WDDM an over-subscribed allocation does not fail, it pages to system memory and crawls.
// (with borrowing - the default with a profile - the prompt path lends cache slots instead; `pf_borrow` is
@@ -5985,7 +6012,7 @@ int main(int argc, char** argv) {
strata::core::detail::HostMemory hm{};
if (cudaGetDevice(&apu_dev) == cudaSuccess && cudaGetDeviceProperties(&apu_prop, apu_dev) == cudaSuccess &&
apu_prop.integrated && strata::core::detail::host_available_memory(hm)) {
- const int64_t host_headroom = 4ll << 30; // the OS and request-time CPU work
+ const int64_t host_headroom = (int64_t) strata::core::uma_desktop_reserve_bytes(strata::platform::total_physical_memory()); // the OS, the desktop and request-time CPU work
const int64_t cache_b = (int64_t) o.expert_cache * blob;
if (cache_b > (int64_t) hm.available - host_headroom)
std::fprintf(stderr, "strata generate: WARNING: this GPU shares system RAM, and the cache above (%.2f GiB) is "