Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

PLE residency policy: admit the table only when RAM holds table + the expert arena + 8 GiB. The 0.2.7 rule (table + 8 GiB) admitted the 26.8 GiB table on a 64 GB Windows PC (RTX 5070, IQ3_S), where the engine then fills a 51 GiB pinned expert arena: the locked table left too little lockable memory and the expert load died with ERROR_WORKING_SET_QUOTA (1453). ple_resident_admit() (include/strata/core/ple_residency.hpp) sizes the arena from the native expert layout; unknown sizes or telemetry admit nothing. Unit test ple_residency_test.

Niko1221472de9cdac50
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c4e6e8a18..d67949f28 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -138,6 +138,9 @@ if(STRATA_BUILD_CONVERSATION_TESTS)
add_executable(conversation_memory_test src/core/conversation_memory_test.cpp src/core/conversation_memory.cpp)
target_include_directories(conversation_memory_test PRIVATE include)
add_test(NAME conversation_memory_test COMMAND conversation_memory_test)
+ add_executable(ple_residency_test src/core/ple_residency_test.cpp)
+ target_include_directories(ple_residency_test PRIVATE include)
+ add_test(NAME ple_residency_test COMMAND ple_residency_test)
add_executable(conversation_file_test src/core/conversation_file_test.cpp src/core/conversation_file.cpp)
target_include_directories(conversation_file_test PRIVATE include)
add_test(NAME conversation_file_test COMMAND conversation_file_test)
diff --git a/include/strata/core/ple_residency.hpp b/include/strata/core/ple_residency.hpp
new file mode 100644
index 000000000..1135b89ec
--- /dev/null
+++ b/include/strata/core/ple_residency.hpp
@@ -0,0 +1,29 @@
+#pragma once
+
+#include <cstdint>
+#include <optional>
+
+#include "strata/core/conversation_memory.hpp"
+
+namespace strata::core {
+
+/// The 0.2.7 residency policy for the PLE (n-gram) table: keep the whole table in RAM, or leave it on direct reads?
+///
+/// The table is not the only thing that wants RAM. The engine copies every routed expert into a pinned host arena at
+/// start (about 51 GiB for IQ3_S), after the table has been opened, and an unbuffered read into that arena needs the
+/// pages locked. A locked table takes its share of the lockable memory first, so on a 64 GB PC (Windows, RTX 5070,
+/// 2026-10-26) the policy that only looked at "table + 8 GiB available" admitted the table and the expert load then
+/// failed with ERROR_WORKING_SET_QUOTA (1453). The table is therefore admitted only when RAM holds the table AND the
+/// arena the engine will still fill AND the headroom, all measured against what is available BEFORE either is
+/// allocated. Unknown memory telemetry, an unknown table size or an unknown arena size admit nothing (the direct
+/// reads of 0.2.6 stay).
+///
+/// `experts_ram_bytes`: the native expert layout's total (every layer's blob x experts), 0 when it is not known.
+inline bool ple_resident_admit(std::optional<uint64_t> available, uint64_t table_bytes, uint64_t experts_ram_bytes,
+ uint64_t headroom_bytes) {
+ if (!available || table_bytes == 0 || experts_ram_bytes == 0) return false;
+ if (table_bytes > UINT64_MAX - experts_ram_bytes) return false;
+ return conversation_memory_admit(available, table_bytes + experts_ram_bytes, headroom_bytes);
+}
+
+} // namespace strata::core
diff --git a/src/core/ple_residency_test.cpp b/src/core/ple_residency_test.cpp
new file mode 100644
index 000000000..4673b7adc
--- /dev/null
+++ b/src/core/ple_residency_test.cpp
@@ -0,0 +1,40 @@
+#include "strata/core/ple_residency.hpp"
+
+#include <cstdio>
+#include <cstdlib>
+#include <limits>
+
+using namespace strata::core;
+namespace {
+int checks = 0;
+void check(bool ok, const char* label) {
+ ++checks;
+ if (!ok) { std::fprintf(stderr, "FAIL: %s\n", label); std::exit(1); }
+}
+constexpr uint64_t GiB = 1ull << 30;
+}
+
+int main() {
+ const uint64_t table = 26ull * GiB + 870ull * (1ull << 20); // IQ3_S's table, 26.8 GiB
+ const uint64_t arena = 51ull * GiB + 100ull * (1ull << 20); // IQ3_S's 51.1 GiB of experts
+ // the owner PC, 2026-10-26: 45.2 GiB available when the table is opened, 63 GiB total. The 0.2.7 first policy
+ // (table + 8 GiB) admitted it and the expert load then failed with error 1453. Not admitted now.
+ check(!ple_resident_admit(45ull * GiB + (1ull << 28), table, arena, 8 * GiB), "64 GB PC with the IQ3_S arena: not admitted");
+ check(!ple_resident_admit(40ull * GiB, table, arena, 8 * GiB), "40 GiB: not admitted");
+ // a PC that holds the table, the arena and 8 GiB
+ check(ple_resident_admit(table + arena + 8 * GiB, table, arena, 8 * GiB), "exactly table + arena + headroom is admitted");
+ check(!ple_resident_admit(table + arena + 8 * GiB - 1, table, arena, 8 * GiB), "one byte below is not");
+ check(ple_resident_admit(240ull * GiB, table, arena, 8 * GiB), "a 256 GB box: admitted");
+ // a smaller model's arena on a mid PC (Q2_0: 34 GiB arena): 26.8 + 34 + 8 = 68.8
+ check(ple_resident_admit(80ull * GiB, table, 34ull * GiB, 8 * GiB), "Q2_0 on 96 GB: admitted");
+ check(!ple_resident_admit(60ull * GiB, table, 34ull * GiB, 8 * GiB), "Q2_0 with 60 GiB available: not admitted");
+ // unknowns fail closed
+ check(!ple_resident_admit({}, table, arena, 8 * GiB), "unknown telemetry");
+ check(!ple_resident_admit(240ull * GiB, 0, arena, 8 * GiB), "unknown table size");
+ check(!ple_resident_admit(240ull * GiB, table, 0, 8 * GiB), "unknown arena size");
+ check(!ple_resident_admit(std::numeric_limits<uint64_t>::max(), table, std::numeric_limits<uint64_t>::max(), 8 * GiB),
+ "sizes cannot overflow");
+ check(!ple_resident_admit(0, table, arena, 0), "zero available");
+ std::printf("ple_residency_test: %d checks passed\n", checks);
+ return 0;
+}
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 45aae75ac..28f6fb1a2 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -29,6 +29,7 @@
#include "strata/core/conversation_file.hpp"
#include "strata/core/session_save_reclaim.hpp"
#include "strata/core/conversation_memory.hpp"
+#include "strata/core/ple_residency.hpp"
#include "strata/core/coupled_draft.hpp"
#include "strata/core/expert_source.hpp"
#include "strata/core/foresight_swap.hpp"
@@ -4065,6 +4066,10 @@ int main(int argc, char** argv) {
// prompt 2125 / 2368 / 2611 tok/s, and the answer md5 was identical in all three. The RAM cost is the
// table itself; the policy needs 8 GiB of headroom on top of it (available_host_bytes accounts for
// cgroups and --memory-limit-mib), an explicit --ple-io always wins, and STRATA_PLE_AUTO_RAM=0 turns it off.
+ // The expert arena is filled AFTER this point and wants its own RAM (51 GiB for IQ3_S), so the table is admitted
+ // only when RAM holds table + arena + headroom (strata/core/ple_residency.hpp). Measured on the 64 GB RTX 5070 PC
+ // (2026-10-26): table + 8 GiB alone admitted it, the table's lock took the lockable memory and the expert load
+ // died with error 1453 (ERROR_WORKING_SET_QUOTA).
bool ple_auto_ram = false;
if (pio.mode == strata::kernels::PleIo::Direct && !o.ple_io_explicit) {
const strata::kernels::PleIoOptions direct_pio = pio;
@@ -4072,7 +4077,13 @@ int main(int argc, char** argv) {
const bool allowed = auto_ram == nullptr || std::atoi(auto_ram) != 0;
const uint64_t need = ple_table.table_bytes();
const auto available = strata::core::available_host_bytes();
- if (allowed && need > 0 && strata::core::conversation_memory_admit(available, need, 8ull << 30)) {
+ uint64_t experts_ram = 0; // every routed expert's blob, the arena the engine copies into RAM at start
+ {
+ const auto& elay = strata::kernels::cpu::expert_layout();
+ for (int64_t l = 0; l < elay.n_layers; ++l)
+ experts_ram += elay.blob_bytes(l) * (uint64_t) elay.n_expert;
+ }
+ if (allowed && need > 0 && strata::core::ple_resident_admit(available, need, experts_ram, 8ull << 30)) {
ple_table.close();
pio.mode = strata::kernels::PleIo::Mmap;
pio.lock = true;