Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

engine: wait up to 20 s for the RAM the expert arena needs, then load with a plainer warning (#4051)

Niko122107d5ecfacb4c
diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hpp
index 2bcfca39b..45ac7f742 100644
--- a/include/strata/core/expert_source.hpp
+++ b/include/strata/core/expert_source.hpp
@@ -36,6 +36,7 @@
#include <cstddef>
#include <condition_variable>
#include <deque>
+#include <functional>
#include <memory>
#include <mutex>
#include <thread>
@@ -94,6 +95,16 @@ bool host_available_memory(HostMemory& m, const std::string& meminfo = "/proc/me
const std::string& self_cgroup = "/proc/self/cgroup",
const std::string& cgroup_root = "/sys/fs/cgroup");
+/// #4051: wait for the RAM an arena needs to come back. An engine that was still exiting (or a program that was ending)
+/// holds memory a few seconds longer than its process; loading into that shortfall sent a 60 GiB host into reclaim for
+/// good. Calls `sample(available)` (false: the RAM cannot be read; the wait ends) every `step_s` seconds, through
+/// `sleep(step_s)`, until `available >= need` or `max_wait_s` seconds have gone. `waited` gets the seconds waited,
+/// `available` the last reading. True when the RAM is there. `progress(waited, available)` is called about every
+/// `note_s` seconds while waiting. `max_wait_s <= 0` is no wait at all.
+bool wait_for_ram(uint64_t need, double max_wait_s, double step_s, double note_s,
+ const std::function<bool(uint64_t&)>& sample, const std::function<void(double)>& sleep,
+ const std::function<void(double, uint64_t)>& progress, double& waited, uint64_t& available);
+
/// Bound a resident budget by RAM and commit capacity after headroom; leave 256 MiB more when clamping.
/// Pass UINT64_MAX for commit when the platform does not report it.
uint64_t clamp_resident_budget(uint64_t requested, uint64_t physical, uint64_t commit, uint64_t headroom);
diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp
index 1b5b11552..41d882ddc 100644
--- a/src/core/expert_source.cpp
+++ b/src/core/expert_source.cpp
@@ -104,6 +104,24 @@ bool pin_depends_on_reclaim(uint64_t free_now, uint64_t bytes, uint64_t reserve)
return bytes > 0 && free_now < bytes + reserve;
}
+bool wait_for_ram(uint64_t need, double max_wait_s, double step_s, double note_s,
+ const std::function<bool(uint64_t&)>& sample, const std::function<void(double)>& sleep,
+ const std::function<void(double, uint64_t)>& progress, double& waited, uint64_t& available) {
+ waited = 0.0;
+ if (!sample(available)) return false;
+ double next_note = 0.0;
+ while (available < need && waited < max_wait_s) {
+ if (progress && waited >= next_note) {
+ progress(waited, available);
+ next_note = waited + note_s;
+ }
+ sleep(step_s);
+ waited += step_s;
+ if (!sample(available)) return false;
+ }
+ return available >= need;
+}
+
bool pin_step_fits(uint64_t available, uint64_t step, uint64_t reserve) {
return available >= step + reserve;
}
@@ -5820,12 +5838,47 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
return false;
}
if (hm.available < need) {
- std::snprintf(buf, sizeof buf,
- "the expert arena needs %.2f GiB of RAM but %.2f GiB is available (%.2f GiB short): the load "
- "may swap or be stopped by the OS. Close other programs (or wait for an engine that is "
- "still exiting), or run with less RAM: --mmap-experts with --resident-budget-gib N",
- (double) need / gib, (double) hm.available / gib, (double) (need - hm.available) / gib);
- ram_warning_ = buf;
+ // #4051: the shortfall is often an engine that was still exiting (the code's own example), a few seconds
+ // from giving its RAM back: wait for it, not for ever, and say so. STRATA_ARENA_RAM_WAIT_S changes the
+ // seconds (0 loads at once, as before this wait). The load then goes on either way - it only warns.
+ double wait_s = 20.0;
+ if (const char* w = std::getenv("STRATA_ARENA_RAM_WAIT_S"); w != nullptr && w[0] != '\0') {
+ char* end = nullptr;
+ const double v = std::strtod(w, &end);
+ if (end != w && *end == '\0' && v >= 0.0 && v <= 3600.0) wait_s = v;
+ }
+ double waited = 0.0;
+ uint64_t avail = hm.available;
+ if (wait_s > 0.0) {
+ const bool came = detail::wait_for_ram(
+ need, wait_s, 0.25, 5.0,
+ [](uint64_t& a) { detail::HostMemory h; if (!detail::host_available_memory(h)) return false; a = h.available; return true; },
+ [](double s) { std::this_thread::sleep_for(std::chrono::duration<double>(s)); },
+ [&](double w, uint64_t a) {
+ std::fprintf(stderr, "strata generate: the expert arena needs %.2f GiB of RAM and %.2f GiB is "
+ "available; waiting up to %.0f s for it (another engine still exiting?) - "
+ "%.0f s so far (STRATA_ARENA_RAM_WAIT_S changes this)\n",
+ (double) need / gib, (double) a / gib, wait_s, w);
+ },
+ waited, avail);
+ if (came)
+ std::fprintf(stderr, "strata generate: %.2f GiB of RAM is available now (after %.1f s): loading the "
+ "expert arena\n", (double) avail / gib, waited);
+ hm.available = avail;
+ }
+ if (hm.available < need) {
+ std::snprintf(buf, sizeof buf,
+ "the expert arena needs %.2f GiB of RAM but %.2f GiB is available (%.2f GiB short%s): the "
+ "load may swap or be stopped by the OS%s. Close other programs, or run with less RAM: "
+ "--mmap-experts with --resident-budget-gib N",
+ (double) need / gib, (double) hm.available / gib, (double) (need - hm.available) / gib,
+ waited > 0.0 ? ", after waiting" : "",
+ hm.available < need / 2 ? " - with this little room the whole machine can stall in "
+ "memory reclaim until it is restarted (#4051); on Linux a "
+ "systemd MemoryHigh= below the RAM keeps one program from "
+ "taking all of it" : "");
+ ram_warning_ = buf;
+ }
}
}
diff --git a/tests/core/file_expert_source_test.cpp b/tests/core/file_expert_source_test.cpp
index d6a274f1a..3f3cf532f 100644
--- a/tests/core/file_expert_source_test.cpp
+++ b/tests/core/file_expert_source_test.cpp
@@ -369,6 +369,38 @@ void test_resident_exchange() {
require(offsets == before, "exchanging back did not restore the plan");
}
+// #4051: the wait for the RAM an arena needs (an engine still exiting gives it back a few seconds late)
+void test_wait_for_ram() {
+ using namespace strata::core::detail;
+ constexpr uint64_t GiB = 1ull << 30;
+ double waited = -1.0;
+ uint64_t avail = 0;
+ int slept = 0, noted = 0;
+ const auto sleeper = [&](double) { ++slept; };
+ const auto note = [&](double, uint64_t) { ++noted; };
+ // enough at once: no wait, no sleep, no note
+ require(wait_for_ram(40 * GiB, 20.0, 0.25, 5.0, [](uint64_t& a) { a = 50 * GiB; return true; }, sleeper, note, waited, avail) &&
+ waited == 0.0 && slept == 0 && noted == 0 && avail == 50 * GiB, "RAM that was there was waited for");
+ // 10 GiB free for a 40 GiB arena, the exiting engine gives its 50 GiB back after 3 s (12 steps of 0.25 s)
+ int reads = 0;
+ slept = noted = 0;
+ require(wait_for_ram(40 * GiB, 20.0, 0.25, 5.0, [&](uint64_t& a) { a = ++reads > 12 ? 60 * GiB : 10 * GiB; return true; },
+ sleeper, note, waited, avail) && waited == 3.0 && slept == 12 && noted == 1 && avail == 60 * GiB,
+ "RAM that came back after 3 s was not waited for");
+ // it never comes: the wait ends at the limit and says so (the load then goes on and warns)
+ slept = noted = 0;
+ require(!wait_for_ram(40 * GiB, 20.0, 0.25, 5.0, [](uint64_t& a) { a = 10 * GiB; return true; }, sleeper, note, waited, avail) &&
+ waited == 20.0 && slept == 80 && noted == 4 && avail == 10 * GiB, "the wait did not end at its limit");
+ // no wait asked for: one reading
+ slept = 0;
+ require(!wait_for_ram(40 * GiB, 0.0, 0.25, 5.0, [](uint64_t& a) { a = 10 * GiB; return true; }, sleeper, note, waited, avail) &&
+ waited == 0.0 && slept == 0, "a zero wait slept");
+ // the RAM cannot be read: the wait ends at once
+ slept = 0;
+ require(!wait_for_ram(40 * GiB, 20.0, 0.25, 5.0, [](uint64_t&) { return false; }, sleeper, note, waited, avail) && slept == 0,
+ "an unreadable RAM was waited for");
+}
+
// #1250: when the page-locked complement is registered in steps, and what a meminfo reading says
void test_pin_pacing() {
using namespace strata::core::detail;
@@ -1014,6 +1046,7 @@ int main(int argc, char** argv) {
test_resident_exchange();
test_resident_memory_budget();
test_pin_pacing();
+ test_wait_for_ram();
test_numa_slice_ranges();
test_cgroup_memory_budget();
test_host_memory();