Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Qwen: the PLE residency policy gives its lock back when the lock is what leaves the automatic conversation cache too little RAM (#4518)

Niko1221587a47443328
diff --git a/include/strata/core/ple_residency.hpp b/include/strata/core/ple_residency.hpp
index b9e3829e2..4e358af0b 100644
--- a/include/strata/core/ple_residency.hpp
+++ b/include/strata/core/ple_residency.hpp
@@ -34,4 +34,15 @@ inline bool ple_auto_ram_allowed(const char* env_value, bool v026) {
return !v026;
}
+/// #4518: the table was locked at start on the policy's word (table + arena + headroom fit), but what the rest of the engine takes after
+/// that (the KV pools, the pinned staging, the GPUs' host buffers) is not known then. On a 128 GB two-GPU Windows PC it left less than the
+/// 5 GiB the automatic conversation cache needs, and the cache turned itself off. The cache is worth more than the lock (parking a 100 K
+/// conversation is 1.1 s against 30 s to read it again, the lock was measured at 2-3% of prompt speed), so once the cache is to be sized:
+/// does giving the lock back make the automatic cache bigger? `available` is the RAM the cache sees now (the table locked), the table's
+/// pages count as available again once unlocked (clean file pages). Unknown telemetry: no.
+inline bool ple_lock_blocks_cache(std::optional<uint64_t> available, uint64_t table_bytes, int64_t min_free_mib) {
+ if (!available || table_bytes == 0 || *available > UINT64_MAX - table_bytes) return false;
+ return auto_conversation_cache_mib(*available + table_bytes, min_free_mib) > auto_conversation_cache_mib(available, min_free_mib);
+}
+
} // namespace strata::core
diff --git a/include/strata/kernels/ngram.hpp b/include/strata/kernels/ngram.hpp
index 9954a0f33..680631afe 100644
--- a/include/strata/kernels/ngram.hpp
+++ b/include/strata/kernels/ngram.hpp
@@ -205,6 +205,10 @@ public:
bool is_open() const;
/// True when `PleIoOptions::lock` was asked for and the OS lock succeeded (false: pages only pre-touched).
bool locked() const;
+ /// Gives a resident lock back (Mmap mode): the pages stay mapped and cached, but the OS may reclaim them again, so the RAM counts as
+ /// available. False when the table was not locked. Used by the residency policy when the table's lock would leave the
+ /// conversation cache no room (#4518).
+ bool unlock();
uint64_t rows() const;
/// Bytes of the whole table (rows x row bytes) while open; 0 when closed. The residency policy in
/// generate.cpp sizes the table with this before it decides to keep it in RAM.
diff --git a/src/core/ple_residency_test.cpp b/src/core/ple_residency_test.cpp
index fedfb9f2d..828f59f12 100644
--- a/src/core/ple_residency_test.cpp
+++ b/src/core/ple_residency_test.cpp
@@ -41,6 +41,20 @@ int main() {
check(!ple_resident_admit(std::numeric_limits<uint64_t>::max(), table, std::numeric_limits<uint64_t>::max(), 8 * GiB),
"sizes cannot overflow");
check(!ple_resident_admit(0, table, arena, 0), "zero available");
+ // #4518: after the load, does the table's lock leave the automatic conversation cache no room? (min free 2560 MiB, cache 40% of the
+ // rest, 8192 MiB at most, 1024 MiB at least)
+ {
+ const int64_t floor_mib = 2560;
+ check(!ple_lock_blocks_cache(std::nullopt, table, floor_mib), "unknown telemetry: leave the lock");
+ check(!ple_lock_blocks_cache(40ull * GiB, 0, floor_mib), "unknown table size: leave the lock");
+ check(ple_lock_blocks_cache(4ull * GiB, table, floor_mib), "the reporter: 4 GiB left with the table locked (cache off), 30 GiB without");
+ check(ple_lock_blocks_cache(10ull * GiB, table, floor_mib), "10 GiB left: cache 1.9 GiB with the lock, 8 GiB without");
+ check(!ple_lock_blocks_cache(40ull * GiB, table, floor_mib), "40 GiB left: the cache is already at its 8 GiB cap");
+ check(!ple_lock_blocks_cache(23ull * GiB, table, floor_mib), "23 GiB: the cap is just reached with the lock");
+ check(ple_lock_blocks_cache(22ull * GiB, table, floor_mib), "22 GiB: the lock costs the cache some of its 8 GiB");
+ check(!ple_lock_blocks_cache(1ull * GiB, 1ull << 20, floor_mib), "a tiny table cannot make a cache");
+ check(!ple_lock_blocks_cache(std::numeric_limits<uint64_t>::max(), table, floor_mib), "no overflow");
+ }
std::printf("ple_residency_test: %d checks passed\n", checks);
return 0;
}
diff --git a/src/kernels/ngram.cpp b/src/kernels/ngram.cpp
index d039805ee..c91e34c06 100644
--- a/src/kernels/ngram.cpp
+++ b/src/kernels/ngram.cpp
@@ -519,6 +519,13 @@ void PleTable::close() {
bool PleTable::is_open() const { return impl_->data != nullptr || impl_->reader.is_open(); }
bool PleTable::locked() const { return impl_->locked; }
+bool PleTable::unlock() {
+ std::lock_guard<std::recursive_mutex> lk(impl_->mu);
+ if (!impl_->locked || impl_->data == nullptr) return false;
+ strata::platform::unlock_resident((void*) impl_->data, impl_->n_rows * (uint64_t) impl_->rb);
+ impl_->locked = false;
+ return true;
+}
const char* PleTable::format() const { return impl_->fmt->name; }
PleIo PleTable::mode() const { return impl_->mode; }
uint64_t PleTable::rows() const { return impl_->n_rows; }
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 59f384285..8020d812e 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -4630,6 +4630,7 @@ int main(int argc, char** argv) {
// is a mapping of the ORIGINAL second GGUF shard, the six weights are already loaded in the arena, and the
// three buffers are the only allocation.
strata::kernels::PleTable ple_table;
+ bool ple_lock_by_policy = false; // the table was locked in RAM by the residency policy (not by --ple-io ram): #4518 may give the lock back
std::vector<float> ple_emb_host((size_t) strata::kernels::NG_N_EMBD);
float* ple_emb_dev = nullptr;
float* ple_scratch = nullptr;
@@ -4688,6 +4689,7 @@ int main(int argc, char** argv) {
std::string ram_err;
if (ple_table.open(o.ple_gguf, ram_err, pio)) {
ple_auto_ram = true;
+ ple_lock_by_policy = ple_table.locked();
std::fprintf(stderr, "strata generate: --ple-io ram chosen by the residency policy (the table is "
"%.1f GiB, the expert arena still to fill %.1f GiB and %.1f GiB of RAM is available; "
"STRATA_PLE_AUTO_RAM=0 keeps the direct reads, --ple-io direct forces them)\n",
@@ -10310,8 +10312,21 @@ int main(int argc, char** argv) {
// its whole prompt again, Reused 0 on 12 of 12). An explicit value, 0 included, is never touched.
if (o.conversation_cache_mib < 0) {
if (o.serve && o.prompt_cache > 0 && o.conversation_cache_slots > 0) {
- o.conversation_cache_mib = strata::core::auto_conversation_cache_mib(strata::core::available_host_bytes(),
- o.conversation_cache_min_free_mib);
+ auto cache_room = strata::core::available_host_bytes();
+ // #4518: the residency policy locked the PLE table before it knew what the rest of the load would take; if the lock is what
+ // leaves the automatic cache too little (or none), the lock is given back (the pages stay cached, the OS may reclaim them)
+ // and the cache is sized again. --ple-io ram asked for by hand and --conversation-cache-mib N are never touched.
+ if (ple_lock_by_policy && ple_table.locked() &&
+ strata::core::ple_lock_blocks_cache(cache_room, ple_table.table_bytes(), o.conversation_cache_min_free_mib)) {
+ const auto before = cache_room;
+ if (ple_table.unlock()) {
+ cache_room = strata::core::available_host_bytes();
+ std::fprintf(stderr, "strata serve: PLE table lock given back: the lock left %.1f GiB of RAM for the conversation cache, "
+ "%.1f GiB without it (the table stays cached; --conversation-cache-mib N and --ple-io ram are kept as asked)\n",
+ before ? (double) *before / (1ull << 30) : 0.0, cache_room ? (double) *cache_room / (1ull << 30) : 0.0);
+ }
+ }
+ o.conversation_cache_mib = strata::core::auto_conversation_cache_mib(cache_room, o.conversation_cache_min_free_mib);
if (o.conversation_cache_mib > 0)
std::fprintf(stderr, "strata serve: conversation cache sized from the free RAM: %lld MiB for up to %d parked "
"conversations, so an agent's next turn reuses its prompt (--conversation-cache-mib N sets it, 0 turns it off)\n",