Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Qwen: the PLE residency policy gives its lock back when the lock is what leaves the automatic conversation cache too little RAM (#4518)
Niko1221587a47443328
diff --git a/include/strata/core/ple_residency.hpp b/include/strata/core/ple_residency.hppindex b9e3829e2..4e358af0b 100644--- a/include/strata/core/ple_residency.hpp+++ b/include/strata/core/ple_residency.hpp@@ -34,4 +34,15 @@ inline bool ple_auto_ram_allowed(const char* env_value, bool v026) {return !v026;}+/// #4518: the table was locked at start on the policy's word (table + arena + headroom fit), but what the rest of the engine takes after+/// that (the KV pools, the pinned staging, the GPUs' host buffers) is not known then. On a 128 GB two-GPU Windows PC it left less than the+/// 5 GiB the automatic conversation cache needs, and the cache turned itself off. The cache is worth more than the lock (parking a 100 K+/// conversation is 1.1 s against 30 s to read it again, the lock was measured at 2-3% of prompt speed), so once the cache is to be sized:+/// does giving the lock back make the automatic cache bigger? `available` is the RAM the cache sees now (the table locked), the table's+/// pages count as available again once unlocked (clean file pages). Unknown telemetry: no.+inline bool ple_lock_blocks_cache(std::optional<uint64_t> available, uint64_t table_bytes, int64_t min_free_mib) {+ if (!available || table_bytes == 0 || *available > UINT64_MAX - table_bytes) return false;+ return auto_conversation_cache_mib(*available + table_bytes, min_free_mib) > auto_conversation_cache_mib(available, min_free_mib);+}+} // namespace strata::corediff --git a/include/strata/kernels/ngram.hpp b/include/strata/kernels/ngram.hppindex 9954a0f33..680631afe 100644--- a/include/strata/kernels/ngram.hpp+++ b/include/strata/kernels/ngram.hpp@@ -205,6 +205,10 @@ public:bool is_open() const;/// True when `PleIoOptions::lock` was asked for and the OS lock succeeded (false: pages only pre-touched).bool locked() const;+ /// Gives a resident lock back (Mmap mode): the pages stay mapped and cached, but the OS may reclaim them again, so the RAM counts as+ /// available. False when the table was not locked. Used by the residency policy when the table's lock would leave the+ /// conversation cache no room (#4518).+ bool unlock();uint64_t rows() const;/// Bytes of the whole table (rows x row bytes) while open; 0 when closed. The residency policy in/// generate.cpp sizes the table with this before it decides to keep it in RAM.diff --git a/src/core/ple_residency_test.cpp b/src/core/ple_residency_test.cppindex fedfb9f2d..828f59f12 100644--- a/src/core/ple_residency_test.cpp+++ b/src/core/ple_residency_test.cpp@@ -41,6 +41,20 @@ int main() {check(!ple_resident_admit(std::numeric_limits<uint64_t>::max(), table, std::numeric_limits<uint64_t>::max(), 8 * GiB),"sizes cannot overflow");check(!ple_resident_admit(0, table, arena, 0), "zero available");+ // #4518: after the load, does the table's lock leave the automatic conversation cache no room? (min free 2560 MiB, cache 40% of the+ // rest, 8192 MiB at most, 1024 MiB at least)+ {+ const int64_t floor_mib = 2560;+ check(!ple_lock_blocks_cache(std::nullopt, table, floor_mib), "unknown telemetry: leave the lock");+ check(!ple_lock_blocks_cache(40ull * GiB, 0, floor_mib), "unknown table size: leave the lock");+ check(ple_lock_blocks_cache(4ull * GiB, table, floor_mib), "the reporter: 4 GiB left with the table locked (cache off), 30 GiB without");+ check(ple_lock_blocks_cache(10ull * GiB, table, floor_mib), "10 GiB left: cache 1.9 GiB with the lock, 8 GiB without");+ check(!ple_lock_blocks_cache(40ull * GiB, table, floor_mib), "40 GiB left: the cache is already at its 8 GiB cap");+ check(!ple_lock_blocks_cache(23ull * GiB, table, floor_mib), "23 GiB: the cap is just reached with the lock");+ check(ple_lock_blocks_cache(22ull * GiB, table, floor_mib), "22 GiB: the lock costs the cache some of its 8 GiB");+ check(!ple_lock_blocks_cache(1ull * GiB, 1ull << 20, floor_mib), "a tiny table cannot make a cache");+ check(!ple_lock_blocks_cache(std::numeric_limits<uint64_t>::max(), table, floor_mib), "no overflow");+ }std::printf("ple_residency_test: %d checks passed\n", checks);return 0;}diff --git a/src/kernels/ngram.cpp b/src/kernels/ngram.cppindex d039805ee..c91e34c06 100644--- a/src/kernels/ngram.cpp+++ b/src/kernels/ngram.cpp@@ -519,6 +519,13 @@ void PleTable::close() {bool PleTable::is_open() const { return impl_->data != nullptr || impl_->reader.is_open(); }bool PleTable::locked() const { return impl_->locked; }+bool PleTable::unlock() {+ std::lock_guard<std::recursive_mutex> lk(impl_->mu);+ if (!impl_->locked || impl_->data == nullptr) return false;+ strata::platform::unlock_resident((void*) impl_->data, impl_->n_rows * (uint64_t) impl_->rb);+ impl_->locked = false;+ return true;+}const char* PleTable::format() const { return impl_->fmt->name; }PleIo PleTable::mode() const { return impl_->mode; }uint64_t PleTable::rows() const { return impl_->n_rows; }diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 59f384285..8020d812e 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -4630,6 +4630,7 @@ int main(int argc, char** argv) {// is a mapping of the ORIGINAL second GGUF shard, the six weights are already loaded in the arena, and the// three buffers are the only allocation.strata::kernels::PleTable ple_table;+ bool ple_lock_by_policy = false; // the table was locked in RAM by the residency policy (not by --ple-io ram): #4518 may give the lock backstd::vector<float> ple_emb_host((size_t) strata::kernels::NG_N_EMBD);float* ple_emb_dev = nullptr;float* ple_scratch = nullptr;@@ -4688,6 +4689,7 @@ int main(int argc, char** argv) {std::string ram_err;if (ple_table.open(o.ple_gguf, ram_err, pio)) {ple_auto_ram = true;+ ple_lock_by_policy = ple_table.locked();std::fprintf(stderr, "strata generate: --ple-io ram chosen by the residency policy (the table is ""%.1f GiB, the expert arena still to fill %.1f GiB and %.1f GiB of RAM is available; ""STRATA_PLE_AUTO_RAM=0 keeps the direct reads, --ple-io direct forces them)\n",@@ -10310,8 +10312,21 @@ int main(int argc, char** argv) {// its whole prompt again, Reused 0 on 12 of 12). An explicit value, 0 included, is never touched.if (o.conversation_cache_mib < 0) {if (o.serve && o.prompt_cache > 0 && o.conversation_cache_slots > 0) {- o.conversation_cache_mib = strata::core::auto_conversation_cache_mib(strata::core::available_host_bytes(),- o.conversation_cache_min_free_mib);+ auto cache_room = strata::core::available_host_bytes();+ // #4518: the residency policy locked the PLE table before it knew what the rest of the load would take; if the lock is what+ // leaves the automatic cache too little (or none), the lock is given back (the pages stay cached, the OS may reclaim them)+ // and the cache is sized again. --ple-io ram asked for by hand and --conversation-cache-mib N are never touched.+ if (ple_lock_by_policy && ple_table.locked() &&+ strata::core::ple_lock_blocks_cache(cache_room, ple_table.table_bytes(), o.conversation_cache_min_free_mib)) {+ const auto before = cache_room;+ if (ple_table.unlock()) {+ cache_room = strata::core::available_host_bytes();+ std::fprintf(stderr, "strata serve: PLE table lock given back: the lock left %.1f GiB of RAM for the conversation cache, "+ "%.1f GiB without it (the table stays cached; --conversation-cache-mib N and --ple-io ram are kept as asked)\n",+ before ? (double) *before / (1ull << 30) : 0.0, cache_room ? (double) *cache_room / (1ull << 30) : 0.0);+ }+ }+ o.conversation_cache_mib = strata::core::auto_conversation_cache_mib(cache_room, o.conversation_cache_min_free_mib);if (o.conversation_cache_mib > 0)std::fprintf(stderr, "strata serve: conversation cache sized from the free RAM: %lld MiB for up to %d parked ""conversations, so an agent's next turn reuses its prompt (--conversation-cache-mib N sets it, 0 turns it off)\n",