Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Keep pinned-memory headroom while the expert arena pins; retry the engine's required pinned buffers once after handing it back (Windows load failure: verify mapped staging allocation failed)
Niko12214f74de9cdbbd
diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hppindex cf5176909..c99dfb867 100644--- a/include/strata/core/expert_source.hpp+++ b/include/strata/core/expert_source.hpp@@ -968,6 +968,7 @@ public:const std::string& note() const { return note_; }/// #633: set by `open` when the RAM available is less than the arena needs (a recommendation; it still loads)const std::string& ram_warning() const { return ram_warning_; }+ uint64_t pinned_arena_bytes() const { return pinned_bytes_; } ///< what the driver registered (the pinned prefix, or the total by need)double load_gib_per_second() const { return gib_per_s_; }// Loader fix: the load, split. `load_seconds()` is the wall clock of the load loop; the other two are// sums over the reader threads (see LoadStats), so on their own they say how much of that wall was spentdiff --git a/include/strata/core/pin_budget.hpp b/include/strata/core/pin_budget.hppnew file mode 100644index 000000000..649667dc4--- /dev/null+++ b/include/strata/core/pin_budget.hpp@@ -0,0 +1,204 @@+// include/strata/core/pin_budget.hpp - pinned (page-locked) host memory is a budget, and the expert arena must not spend all of it.+//+// Windows (WDDM) lets a process page-lock about half the RAM for the GPU (the shared-memory segment). The expert arena+// is registered slice by slice until the driver refuses one, which leaves NOTHING for the pinned/mapped buffers the+// engine allocates after it: the verify window's staging, the session's pool staging, the KV copy's tables. Those were+// only a few MiB, and their failure ended the load ("verify: mapped staging allocation failed", exit 1) on a PC with+// 64 GB of free RAM.+//+// So the engine takes two reserves BEFORE the arena pins anything and gives them back after it:+// * the SOFT reserve is released right after the arena is placed: the prompt path's staging and everything else+// that already degrades to pageable memory then finds room, as it always did;+// * the HARD reserve (the buffers whose failure ended the load) stays held until the first such allocation fails or+// the engine is up; the allocation then releases it and retries once (host_alloc_guarded).+// A reserve that cannot be had is shrunk, then skipped: this never fails a start.+//+// Defaults: Windows 768 + 192 MiB; Linux off (the NVIDIA driver there pins past RLIMIT_MEMLOCK and has no such+// ceiling; checked with a 4 GiB cudaHostRegister under `ulimit -l 1 GiB`). STRATA_PIN_RESERVE_MIB=N sets the soft+// reserve (0 = off) on any OS; the hard one is a fifth of it, at least 64 MiB.+//+// STRATA_TEST_PIN_BUDGET_MIB=N is a TEST knob: it makes the engine's own pinned allocations and registrations that go+// through these wrappers (the arena's, the verify window's, the session's, the KV copy's) fail once N MiB are+// outstanding, which is what the WDDM ceiling does on Windows. It lets Linux reproduce the exhaustion.+#pragma once++#include <cuda_runtime.h>++#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <map>+#include <mutex>+#include <string>++namespace strata::core {++namespace pin_detail {++struct State {+ std::mutex mu;+ uint64_t sim_budget = 0; // bytes; 0 = no simulated ceiling+ uint64_t sim_used = 0;+ uint64_t sim_peak = 0;+ std::map<const void*, uint64_t> sim_ptrs;+ void* soft = nullptr;+ uint64_t soft_bytes = 0;+ void* hard = nullptr;+ uint64_t hard_bytes = 0;+ bool taken = false;+ uint64_t arena_pinned = 0; // what the arena registered (for the hint)+ int hard_retries = 0;+ State() {+ if (const char* e = std::getenv("STRATA_TEST_PIN_BUDGET_MIB"); e != nullptr && *e != '\0')+ sim_budget = (uint64_t) std::strtoull(e, nullptr, 10) << 20;+ }+};+inline State& st() {+ static State s;+ return s;+}+// caller holds the lock+inline bool sim_charge(State& s, const void* p, uint64_t bytes) {+ if (s.sim_budget == 0) return true;+ if (s.sim_used + bytes > s.sim_budget) return false;+ s.sim_used += bytes;+ if (s.sim_used > s.sim_peak) s.sim_peak = s.sim_used;+ if (p != nullptr) s.sim_ptrs[p] = bytes;+ return true;+}+inline void sim_uncharge(State& s, const void* p) {+ if (s.sim_budget == 0) return;+ auto it = s.sim_ptrs.find(p);+ if (it == s.sim_ptrs.end()) return;+ s.sim_used -= it->second;+ s.sim_ptrs.erase(it);+}++inline cudaError_t alloc_locked(State& s, void** p, size_t bytes, unsigned flags) {+ if (s.sim_budget != 0 && s.sim_used + bytes > s.sim_budget) return cudaErrorMemoryAllocation;+ const cudaError_t e = cudaHostAlloc(p, bytes, flags);+ if (e == cudaSuccess) sim_charge(s, *p, bytes);+ return e;+}+inline void free_locked(State& s, void* p) {+ if (p == nullptr) return;+ (void) cudaFreeHost(p);+ sim_uncharge(s, p);+}++inline uint64_t soft_reserve_default_mib() {+ if (const char* e = std::getenv("STRATA_PIN_RESERVE_MIB"); e != nullptr && *e != '\0')+ return (uint64_t) std::strtoull(e, nullptr, 10);+#if defined(_WIN32)+ return 768;+#else+ return st().sim_budget != 0 ? 768 : 0; // Linux: only when the test ceiling is on+#endif+}+} // namespace pin_detail++/// cudaHostRegister through the (test) budget.+inline cudaError_t pin_register(void* p, size_t bytes, unsigned flags) {+ auto& s = pin_detail::st();+ if (s.sim_budget == 0) return cudaHostRegister(p, bytes, flags);+ std::lock_guard<std::mutex> g(s.mu);+ if (s.sim_used + bytes > s.sim_budget) return cudaErrorMemoryAllocation;+ const cudaError_t e = cudaHostRegister(p, bytes, flags);+ if (e == cudaSuccess) pin_detail::sim_charge(s, p, bytes);+ return e;+}+inline cudaError_t pin_unregister(void* p) {+ auto& s = pin_detail::st();+ if (s.sim_budget != 0) {+ std::lock_guard<std::mutex> g(s.mu);+ pin_detail::sim_uncharge(s, p);+ }+ return cudaHostUnregister(p);+}++/// Take the reserves (call before the expert arena pins). Never fails.+inline void pin_reserve_take() {+ auto& s = pin_detail::st();+ std::lock_guard<std::mutex> g(s.mu);+ if (s.taken) return;+ s.taken = true;+ const uint64_t soft_mib = pin_detail::soft_reserve_default_mib();+ if (soft_mib == 0) return;+ const uint64_t hard_mib = soft_mib / 4 > 64 ? soft_mib / 4 : 64;+ // the hard one first: it is the one whose absence ends a load. Shrink by halves down to 16 MiB.+ auto take = [&](void*& p, uint64_t& got, uint64_t mib) {+ for (; mib >= 16; mib /= 2) {+ void* q = nullptr;+ if (pin_detail::alloc_locked(s, &q, (size_t) (mib << 20), cudaHostAllocPortable) == cudaSuccess) {+ p = q;+ got = mib << 20;+ return;+ }+ (void) cudaGetLastError();+ }+ };+ take(s.hard, s.hard_bytes, hard_mib);+ take(s.soft, s.soft_bytes, soft_mib);+ std::fprintf(stderr, "strata: pinned-memory headroom: holding %llu MiB for the buffers allocated after the expert arena "+ "(released after it: %llu MiB; kept until needed: %llu MiB; STRATA_PIN_RESERVE_MIB changes it)\n",+ (unsigned long long) ((s.soft_bytes + s.hard_bytes) >> 20), (unsigned long long) (s.soft_bytes >> 20),+ (unsigned long long) (s.hard_bytes >> 20));+ if (s.soft_bytes + s.hard_bytes < ((soft_mib + hard_mib) << 20))+ std::fprintf(stderr, "strata: WARNING: pinned-memory budget is already nearly spent before the expert arena "+ "(got %llu of %llu MiB of headroom)\n",+ (unsigned long long) ((s.soft_bytes + s.hard_bytes) >> 20), (unsigned long long) (soft_mib + hard_mib));+}+/// The arena is placed: hand the soft reserve back.+inline void pin_reserve_release_soft() {+ auto& s = pin_detail::st();+ std::lock_guard<std::mutex> g(s.mu);+ if (s.soft == nullptr) return;+ pin_detail::free_locked(s, s.soft);+ s.soft = nullptr;+ std::fprintf(stderr, "strata: pinned-memory headroom: %llu MiB handed back after the expert arena\n",+ (unsigned long long) (s.soft_bytes >> 20));+ s.soft_bytes = 0;+}+/// The engine is up (or the hard allocation needs it): hand the hard reserve back.+inline void pin_reserve_release_hard() {+ auto& s = pin_detail::st();+ std::lock_guard<std::mutex> g(s.mu);+ if (s.hard == nullptr) return;+ pin_detail::free_locked(s, s.hard);+ s.hard = nullptr;+ s.hard_bytes = 0;+}+inline void pin_note_arena(uint64_t pinned_bytes) { pin_detail::st().arena_pinned = pinned_bytes; }+inline uint64_t pin_sim_peak() { return pin_detail::st().sim_peak; }++/// cudaHostAlloc for a buffer the engine cannot run without. On a refusal it hands the reserves back and tries once+/// more; the caller still reports a second refusal.+inline cudaError_t host_alloc_guarded(void** p, size_t bytes, unsigned flags) {+ auto& s = pin_detail::st();+ {+ std::lock_guard<std::mutex> g(s.mu);+ const cudaError_t e = pin_detail::alloc_locked(s, p, bytes, flags);+ if (e == cudaSuccess) return e;+ (void) cudaGetLastError();+ if (s.hard == nullptr && s.soft == nullptr) return e;+ if (s.soft != nullptr) { pin_detail::free_locked(s, s.soft); s.soft = nullptr; s.soft_bytes = 0; }+ if (s.hard != nullptr) { pin_detail::free_locked(s, s.hard); s.hard = nullptr; s.hard_bytes = 0; }+ ++s.hard_retries;+ std::fprintf(stderr, "strata: pinned-memory headroom: a %llu KiB pinned buffer was refused; the held headroom is handed back "+ "and the allocation retried\n", (unsigned long long) (bytes >> 10));+ const cudaError_t e2 = pin_detail::alloc_locked(s, p, bytes, flags);+ if (e2 != cudaSuccess) (void) cudaGetLastError();+ return e2;+ }+}++/// The sentence every hard failure ends with, so the app can recognise it in the log.+inline std::string pin_exhausted_hint() {+ auto& s = pin_detail::st();+ const uint64_t gib = s.arena_pinned >> 30;+ std::string t = "[pinned-memory-exhausted] the OS gave no more page-locked memory";+ if (gib > 2) t += "; start again with STRATA_ARENA_PIN_GIB=" + std::to_string(gib - 2) + " (pin less of the expert arena) or with the low-RAM option";+ return t;+}++} // namespace strata::corediff --git a/src/core/layer.cpp b/src/core/layer.cppindex 1e6b3b930..fa421e4e2 100644--- a/src/core/layer.cpp+++ b/src/core/layer.cpp@@ -1,6 +1,7 @@// src/core/layer.cpp - the GDN layer, composed. See the header for the operation order and its traps.#include "strata/core/layer.hpp"+#include "strata/core/pin_budget.hpp"#include "strata/core/native_dense.hpp"#include "strata/core/native_head.hpp"#include "strata/kernels/bf16_bits.hpp"@@ -788,9 +789,9 @@ uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, Qst.attention_status = c.take<int32_t>(1);st.pos_dev = c.take<int32_t>((uint64_t) s.n_head);// the pinned staging the uploads copy FROM - see the note on `host_step` in the header- if (cudaHostAlloc((void**) &st.host_step, strata::kernels::qsa_step_bytes() + sizeof(int32_t),+ if (strata::core::host_alloc_guarded((void**) &st.host_step, strata::kernels::qsa_step_bytes() + sizeof(int32_t),cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess ||- cudaHostAlloc((void**) &st.host_pos, (size_t) s.n_head * 4, cudaHostAllocMapped | cudaHostAllocPortable) !=+ strata::core::host_alloc_guarded((void**) &st.host_pos, (size_t) s.n_head * 4, cudaHostAllocMapped | cudaHostAllocPortable) !=cudaSuccess) {return 0; // the caller sees a zero byte count; a half-built state is worse than none}@@ -802,7 +803,7 @@ uint64_t qsa_state_init(const ModelGeometry& g, int64_t max_cells, void* base, Qconst uint64_t bytes = (uint64_t) pages * strata::kernels::kv_block_bytes(s, qsa_kv_format(st)) + 4 * 256;uint8_t* h = nullptr;uint8_t* d = nullptr;- if (cudaHostAlloc((void**) &h, bytes, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess ||+ if (strata::core::host_alloc_guarded((void**) &h, bytes, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess ||cudaHostGetDevicePointer((void**) &d, h, 0) != cudaSuccess) {// under WSL the NVIDIA driver pins only ~1 GiB in all, which is less than 128K of 8-bit KV needsif (p.mode == 1) std::fprintf(stderr, "strata: KV streaming: cannot pin %.2f GiB of RAM for a layer's KV copy "diff --git a/src/core/mtp.cpp b/src/core/mtp.cppindex c36841c6c..2dbe1a67b 100644--- a/src/core/mtp.cpp+++ b/src/core/mtp.cpp@@ -1,5 +1,6 @@// src/core/mtp.cpp - see include/strata/core/mtp.hpp.#include "strata/core/mtp.hpp"+#include "strata/core/pin_budget.hpp"#include "strata/core/coupled_draft.hpp"#include "strata/core/spec_prob.hpp"#include "strata/core/on_device.hpp"@@ -92,7 +93,7 @@ struct Bump {};bool mapped(size_t bytes, void** h, void** d) {- if (cudaHostAlloc(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false;+ if (strata::core::host_alloc_guarded(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false;std::memset(*h, 0, bytes);return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;}diff --git a/src/core/pinned.cu b/src/core/pinned.cuindex 42c18b760..45204f4af 100644--- a/src/core/pinned.cu+++ b/src/core/pinned.cu@@ -1,6 +1,7 @@// src/core/pinned.cu - P2.S1: the pinned host arena and the parallel expert load.#include "strata/core/pinned.hpp"#include "strata/platform/memory.hpp"+#include "strata/core/pin_budget.hpp"#include <cuda_runtime.h>@@ -420,7 +421,7 @@ PinnedArena::PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds,}const bool capped = cap > 0 && cap < bytes && bounds.size() >= 2;const cudaError_t e = capped ? cudaSuccess :- cudaHostRegister(base, (size_t) bytes, cudaHostRegisterPortable | cudaHostRegisterMapped);+ pin_register(base, (size_t) bytes, cudaHostRegisterPortable | cudaHostRegisterMapped);if (!capped && e == cudaSuccess) {note = "cudaHostRegister PORTABLE ok; " + note;registered_bytes = bytes;@@ -440,7 +441,7 @@ PinnedArena::PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds,for (size_t i = 0; i + 1 < bounds.size(); ++i) {const uint64_t off = bounds[i], n = bounds[i + 1] - bounds[i];if (limited && (off > limit || n > limit - off)) break;- if (cudaHostRegister((uint8_t*) base + off, (size_t) n, cudaHostRegisterPortable | cudaHostRegisterMapped) != cudaSuccess) {+ if (pin_register((uint8_t*) base + off, (size_t) n, cudaHostRegisterPortable | cudaHostRegisterMapped) != cudaSuccess) {(void) cudaGetLastError();break;}@@ -546,7 +547,7 @@ PinnedArena::PinnedArena(uint64_t bytes, ByNeed, const std::vector<std::pair<uinif (it != done.end() && it->first < b) b = it->first;if (a >= b) continue;if (cap > 0 && total + (b - a) > cap) { capped = true; break; }- if (cudaHostRegister((uint8_t*) base + a, (size_t) (b - a), cudaHostRegisterPortable | cudaHostRegisterMapped) !=+ if (pin_register((uint8_t*) base + a, (size_t) (b - a), cudaHostRegisterPortable | cudaHostRegisterMapped) !=cudaSuccess) {(void) cudaGetLastError();refused = 1;@@ -602,7 +603,7 @@ void PinnedArena::register_slices(std::atomic<int>& ready) {int i = 0;for (; base != nullptr && i < n; ++i) {const uint64_t off = bounds_[(size_t) i], len = bounds_[(size_t) i + 1] - off;- if (cudaHostRegister((uint8_t*) base + off, (size_t) len, cudaHostRegisterPortable | cudaHostRegisterMapped) !=+ if (pin_register((uint8_t*) base + off, (size_t) len, cudaHostRegisterPortable | cudaHostRegisterMapped) !=cudaSuccess) {(void) cudaGetLastError();break;@@ -631,7 +632,7 @@ void PinnedArena::register_slices(std::atomic<int>& ready) {PinnedArena::~PinnedArena() {if (base && by_need) {for (const auto& r : locked_ranges) strata::platform::unlock_resident((uint8_t*) base + r.first, r.second);- for (const auto& r : pin_ranges) cudaHostUnregister((uint8_t*) base + r.first);+ for (const auto& r : pin_ranges) pin_unregister((uint8_t*) base + r.first);release(mapping_base ? mapping_base : base, mapping_bytes ? mapping_bytes : capacity);base = nullptr;mapping_base = nullptr;@@ -641,9 +642,9 @@ PinnedArena::~PinnedArena() {if (base) {if (locked_bytes) strata::platform::unlock_resident((uint8_t*) base + (slice_bytes ? registered_bytes : 0), locked_bytes);if (slice_bytes) {- for (uint64_t off : slice_starts) cudaHostUnregister((uint8_t*) base + off);+ for (uint64_t off : slice_starts) pin_unregister((uint8_t*) base + off);} else {- cudaHostUnregister(base);+ pin_unregister(base);}release(mapping_base ? mapping_base : base, mapping_bytes ? mapping_bytes : capacity);base = nullptr;diff --git a/src/core/session.cpp b/src/core/session.cppindex 157c4cc4a..d3aea7da9 100644--- a/src/core/session.cpp+++ b/src/core/session.cpp@@ -1,5 +1,6 @@// src/core/session.cpp - one token through all 48 layers. See the header for why the graphs are per-layer.#include "strata/core/session.hpp"+#include "strata/core/pin_budget.hpp"#include "strata/kernels/mrope.hpp"#include "strata/core/progress.hpp"@@ -541,7 +542,7 @@ bool SessionLoopScratch::init(size_t parts_bytes_in, std::string& err) {}parts_bytes = parts_bytes_in;// MAPPED as well as pinned: the token graph's handoff kernel reads it through its device pointer.- if (cudaHostAlloc((void**) &y_miss, parts_bytes, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {+ if (strata::core::host_alloc_guarded((void**) &y_miss, parts_bytes, cudaHostAllocMapped | cudaHostAllocPortable) != cudaSuccess) {err = "SessionLoopScratch: cudaHostAlloc for the pool's staging failed";return false;}diff --git a/src/core/verify.cpp b/src/core/verify.cppindex 9a9830fe7..ee75bc59b 100644--- a/src/core/verify.cpp+++ b/src/core/verify.cpp@@ -2,6 +2,7 @@#include "strata/core/batch_rows.hpp"#include "strata/kernels/gfx_arch.hpp"#include "strata/core/verify.hpp"+#include "strata/core/pin_budget.hpp"#if !defined(STRATA_USE_HIP) && !defined(STRATA_HIP_GFX906)#include "strata/core/host_graph_launch.hpp"#endif@@ -340,7 +341,7 @@ bool mapped(size_t bytes, void** h, void** d) {return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;}#endif- if (cudaHostAlloc(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false; // multi-GPU: portable only with a peer - the peer card writes its rows into them+ if (strata::core::host_alloc_guarded(h, bytes, cudaHostAllocMapped | (peer_portable() ? cudaHostAllocPortable : 0)) != cudaSuccess) return false; // multi-GPU: portable only with a peer - the peer card writes its rows into themstd::memset(*h, 0, bytes);return cudaHostGetDevicePointer(d, *h, 0) == cudaSuccess;}@@ -662,7 +663,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&mapped(64, (void**) &h_flagB_, (void**) &m_flagB_) &&mapped(64, (void**) &h_plan_err_, (void**) &m_plan_err_) &&mapped(T * K * N * 4, (void**) &h_ymiss_, (void**) &m_ymiss_);- if (!ok) { err = "verify: mapped staging allocation failed"; return false; }+ if (!ok) { err = "verify: mapped staging allocation failed " + strata::core::pin_exhausted_hint(); return false; }// the GPU plan: counts(4) | start(cap+1) | dst(cap) | tok(cap) | pad | ptr(cap u64) | ptr2(cap u64) | start2(cap+1){const int64_t cap = (int64_t) (T * K);@@ -670,7 +671,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&const int64_t ptr_off = (i32 + 1) & ~1ll;plan_i32_ = ptr_off + 4 * cap + (cap + 1) + 1;if (!mapped((size_t) plan_i32_ * 4 * 2 + 64, (void**) &h_plan_, (void**) &m_plan_)) {- err = "verify: mapped plan allocation failed";+ err = "verify: mapped plan allocation failed " + strata::core::pin_exhausted_hint();return false;}sink_.counts = h_plan_;diff --git a/src/program/generate.cpp b/src/program/generate.cppindex fa8eb17c1..4c9aab0eb 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -31,6 +31,7 @@#include "strata/core/expert_source.hpp"#include "strata/core/foresight_swap.hpp"#include "strata/core/pinned.hpp"+#include "strata/core/pin_budget.hpp"#include "strata/core/remote_experts.hpp"#include "strata/core/on_device.hpp"#include "strata/core/peer_experts.hpp"@@ -5223,8 +5224,15 @@ int main(int argc, char** argv) {arena_src.set_pin_priority(pin_cand, pin_nonres);}}- if (!arena_src.open(o.pack, g.n_layers, g.n_expert, /*threads=*/6, err, pin_limit,- o.shared_expert_arena)) {+ // The arena pins until the OS refuses; on Windows that leaves no page-locked memory for the small buffers the+ // engine allocates after it (the verify window's staging: "mapped staging allocation failed", exit 1). Keep a+ // reserve while the arena pins and hand it back after (pin_budget.hpp).+ strata::core::pin_reserve_take();+ const bool arena_ok = arena_src.open(o.pack, g.n_layers, g.n_expert, /*threads=*/6, err, pin_limit,+ o.shared_expert_arena);+ strata::core::pin_note_arena(arena_src.pinned_arena_bytes());+ strata::core::pin_reserve_release_soft();+ if (!arena_ok) {std::fprintf(stderr, "strata generate: %s\n", err.c_str());return 1;}@@ -10027,6 +10035,7 @@ int main(int argc, char** argv) {}}).detach();}+ strata::core::pin_reserve_release_hard(); // the load is over: nothing else needs the headroomstd::printf("READY %lld stop\n", (long long) o.max_context); // "stop": this engine honours STOPstd::fflush(stdout);std::string line;