Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Linux integrated GPU: the auto expert cache counts the RAM once (cache <= RAM available - desktop reserve); the APU warning uses the same reserve
Niko1221f8019539926f
diff --git a/CMakeLists.txt b/CMakeLists.txtindex fa275e8be..1a7691f43 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1413,6 +1413,9 @@ if(STRATA_BUILD_TESTS)add_executable(arch_defaults_test src/core/arch_defaults_test.cpp src/core/arch_defaults.cpp)target_include_directories(arch_defaults_test PRIVATE include)add_test(NAME arch_defaults_test COMMAND arch_defaults_test)+ add_executable(uma_cache_test src/core/uma_cache_test.cpp)+ target_include_directories(uma_cache_test PRIVATE include)+ add_test(NAME uma_cache_test COMMAND uma_cache_test)add_executable(suffix_drafter_test src/spec/suffix_drafter_test.cpp)target_link_libraries(suffix_drafter_test PRIVATE strata_spec)add_test(NAME suffix_drafter_test COMMAND suffix_drafter_test)diff --git a/include/strata/core/uma_cache.hpp b/include/strata/core/uma_cache.hppnew file mode 100644index 000000000..50b929cd7--- /dev/null+++ b/include/strata/core/uma_cache.hpp@@ -0,0 +1,34 @@+// include/strata/core/uma_cache.hpp - sizing the GPU expert cache on a device whose memory IS the system RAM.+//+// On a Strix Halo (or any APU) "free VRAM" is the GPU-addressable pool: the BIOS carve-out plus the GTT, up to nearly all of the RAM when+// the kernel was told ttm.pages_limit (docs/STRIX_HALO.md recommends 112 GiB of a 128 GiB box). The expert arena (every expert, in RAM) is+// already counted in what the OS has used, but the cache copies experts INTO that same RAM, so a cache sized from the pool alone can ask for+// more bytes than the OS has left - the OOM killer then takes the engine, or the desktop. The auto cache is therefore sized from+// min(what the GPU says is free, the RAM the OS says is available - a reserve for the desktop). Only the AUTO size: --expert-cache N is+// kept as asked (the engine warns). STRATA_UMA_RESERVE_GIB=N sets the reserve (0 = no reserve: the old sizing).+#pragma once++#include <cstdint>+#include <cstdlib>++namespace strata::core {++/// The RAM an integrated GPU's expert cache leaves to the OS and the desktop: STRATA_UMA_RESERVE_GIB when set (0 = none), else the larger of+/// 8 GiB and 8% of the installed RAM (a desktop session with a browser needs ~6 GiB; 8% is ~10 GiB on 128 GB).+inline uint64_t uma_desktop_reserve_bytes(uint64_t total_ram, const char* env_gib = std::getenv("STRATA_UMA_RESERVE_GIB")) {+ constexpr uint64_t GiB = 1ull << 30;+ if (env_gib != nullptr && env_gib[0] != '\0') {+ char* end = nullptr;+ const double v = std::strtod(env_gib, &end);+ if (end != env_gib && v >= 0.0) return (uint64_t) (v * (double) GiB);+ }+ const uint64_t pct = total_ram / 100 * 8;+ return pct > 8 * GiB ? pct : 8 * GiB;+}++/// The bytes the auto cache may use: the RAM available now, less the reserve (0 when that is already gone).+inline uint64_t uma_cache_room_bytes(uint64_t available, uint64_t reserve) {+ return available > reserve ? available - reserve : 0;+}++} // namespace strata::corediff --git a/src/core/uma_cache_test.cpp b/src/core/uma_cache_test.cppnew file mode 100644index 000000000..249d4fe8e--- /dev/null+++ b/src/core/uma_cache_test.cpp@@ -0,0 +1,32 @@+// src/core/uma_cache_test.cpp - the RAM reserve of an integrated GPU's auto expert cache (no GPU needed).+#include "strata/core/uma_cache.hpp"++#include <cstdio>++static int fails = 0;+#define CHECK(c) do { if (!(c)) { std::printf("FAIL line %d: %s\n", __LINE__, #c); ++fails; } } while (0)++int main() {+ using namespace strata::core;+ constexpr uint64_t GiB = 1ull << 30;+ // the larger of 8 GiB and 8% of the RAM+ CHECK(uma_desktop_reserve_bytes(32 * GiB, nullptr) == 8 * GiB);+ CHECK(uma_desktop_reserve_bytes(64 * GiB, nullptr) == 8 * GiB); // 8% = 5.1 GiB+ CHECK(uma_desktop_reserve_bytes(128 * GiB, nullptr) > 10 * GiB); // 8% = 10.2 GiB+ CHECK(uma_desktop_reserve_bytes(128 * GiB, nullptr) < 11 * GiB);+ CHECK(uma_desktop_reserve_bytes(0, nullptr) == 8 * GiB); // RAM unknown: the floor+ // the person's own figure, 0 included+ CHECK(uma_desktop_reserve_bytes(128 * GiB, "0") == 0);+ CHECK(uma_desktop_reserve_bytes(128 * GiB, "4") == 4 * GiB);+ CHECK(uma_desktop_reserve_bytes(128 * GiB, "2.5") == 2 * GiB + GiB / 2);+ CHECK(uma_desktop_reserve_bytes(128 * GiB, "") > 10 * GiB); // empty: not set+ CHECK(uma_desktop_reserve_bytes(64 * GiB, "x") == 8 * GiB); // not a number: ignored+ CHECK(uma_desktop_reserve_bytes(64 * GiB, "-3") == 8 * GiB);+ // the room+ CHECK(uma_cache_room_bytes(100 * GiB, 10 * GiB) == 90 * GiB);+ CHECK(uma_cache_room_bytes(10 * GiB, 10 * GiB) == 0);+ CHECK(uma_cache_room_bytes(5 * GiB, 10 * GiB) == 0);+ CHECK(uma_cache_room_bytes(5 * GiB, 0) == 5 * GiB);+ if (fails == 0) std::printf("uma_cache_test: all checks passed\n");+ return fails == 0 ? 0 : 1;+}diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 3cef8621c..b317b68ad 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -33,6 +33,7 @@#include "strata/core/foresight_swap.hpp"#include "strata/core/pinned.hpp"#include "strata/core/pin_budget.hpp"+#include "strata/core/uma_cache.hpp"#include "strata/core/remote_experts.hpp"#include "strata/core/on_device.hpp"#include "strata/core/peer_experts.hpp"@@ -5931,6 +5932,32 @@ int main(int argc, char** argv) {(void) cudaGetLastError();free_b = strata::core::device_free_bytes();}+#if defined(STRATA_USE_HIP) && !defined(_WIN32)+ { // An integrated GPU's "free VRAM" is system RAM (the BIOS carve-out plus the GTT pool, nearly all of the RAM when the kernel was told+ // ttm.pages_limit), and the cache copies experts into that same RAM on top of the arena: sized from the pool alone it can ask for+ // more than the OS has left, and the OOM killer takes the engine or the desktop. So the AUTO size counts that RAM once: the+ // cache gets at most what the OS reports available less a reserve for the desktop (include/strata/core/uma_cache.hpp).+ // A recommendation: --expert-cache N is kept as asked, STRATA_UMA_RESERVE_GIB=0 gives the old sizing.+ cudaDeviceProp up{};+ int ud = 0;+ strata::core::detail::HostMemory uh{};+ if (cudaGetDevice(&ud) == cudaSuccess && cudaGetDeviceProperties(&up, ud) == cudaSuccess && up.integrated &&+ strata::core::detail::host_available_memory(uh)) {+ const uint64_t reserve = strata::core::uma_desktop_reserve_bytes(strata::platform::total_physical_memory());+ const uint64_t room = strata::core::uma_cache_room_bytes((uint64_t) uh.available, reserve);+ if (reserve > 0 && room < (uint64_t) free_b) {+ std::fprintf(stderr, "strata generate: expert cache auto: this GPU shares system RAM: %.2f GiB free on the GPU, but the OS has "+ "%.2f GiB available; %.2f GiB stay for the desktop, so the cache is sized from %.2f GiB "+ "(STRATA_UMA_RESERVE_GIB changes the reserve, --expert-cache N sets the cache)\n",+ (double) free_b / 1073741824.0, (double) uh.available / 1073741824.0,+ (double) reserve / 1073741824.0, (double) room / 1073741824.0);+ free_b = (size_t) room;+ }+ } else {+ (void) cudaGetLastError();+ }+ }+#endif// Plan v0.3 P5: the batched prompt path's chunk buffers are allocated later, so they are reserved here -// under WDDM an over-subscribed allocation does not fail, it pages to system memory and crawls.// (with borrowing - the default with a profile - the prompt path lends cache slots instead; `pf_borrow` is@@ -5985,7 +6012,7 @@ int main(int argc, char** argv) {strata::core::detail::HostMemory hm{};if (cudaGetDevice(&apu_dev) == cudaSuccess && cudaGetDeviceProperties(&apu_prop, apu_dev) == cudaSuccess &&apu_prop.integrated && strata::core::detail::host_available_memory(hm)) {- const int64_t host_headroom = 4ll << 30; // the OS and request-time CPU work+ const int64_t host_headroom = (int64_t) strata::core::uma_desktop_reserve_bytes(strata::platform::total_physical_memory()); // the OS, the desktop and request-time CPU workconst int64_t cache_b = (int64_t) o.expert_cache * blob;if (cache_b > (int64_t) hm.available - host_headroom)std::fprintf(stderr, "strata generate: WARNING: this GPU shares system RAM, and the cache above (%.2f GiB) is "