Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Engine: keep the PLE table resident in RAM when the PC can afford it (opt-out STRATA_PLE_AUTO_RAM=0); measured on p620 R9700 IQ3_S: 8K prompt PLE 845 -> 132 ms, 2125 -> 2611 tok/s, answer md5 identical
Niko12213b34b6c7b29c
diff --git a/include/strata/kernels/ngram.hpp b/include/strata/kernels/ngram.hppindex 4452b4f6d..d25a1eeec 100644--- a/include/strata/kernels/ngram.hpp+++ b/include/strata/kernels/ngram.hpp@@ -202,6 +202,9 @@ public:/// True when `PleIoOptions::lock` was asked for and the OS lock succeeded (false: pages only pre-touched).bool locked() const;uint64_t rows() const;+ /// Bytes of the whole table (rows x row bytes) while open; 0 when closed. The residency policy in+ /// generate.cpp sizes the table with this before it decides to keep it in RAM.+ uint64_t table_bytes() const;/// "IQ4_NL" or "F8_E4M3" (a GGUF from tools/ple_fp8_pack.py: type I8, strata.ple.format = f8_e4m3).const char* format() const;diff --git a/src/kernels/ngram.cpp b/src/kernels/ngram.cppindex 07636d0f2..c32dc786a 100644--- a/src/kernels/ngram.cpp+++ b/src/kernels/ngram.cpp@@ -469,6 +469,7 @@ bool PleTable::locked() const { return impl_->locked; }const char* PleTable::format() const { return impl_->fmt->name; }PleIo PleTable::mode() const { return impl_->mode; }uint64_t PleTable::rows() const { return impl_->n_rows; }+uint64_t PleTable::table_bytes() const { return impl_->n_rows * (uint64_t) impl_->rb; }uint64_t PleTable::bytes_read() const { return impl_->bytes_read; }void PleTable::read_row(uint32_t row, float* out160) const {diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 0b153ee16..45aae75ac 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -4058,6 +4058,44 @@ int main(int argc, char** argv) {std::fprintf(stderr, "strata generate: %s\n", err.c_str());return 1;}+ // 0.2.7 (R9700 measurement): keep the table resident in RAM when the PC can afford it. The rows of a+ // prompt token are random 90 B reads in a 28.8 GB table, so even a cached direct read pays ~100 us a+ // token and a mapped read pays a minor fault a row. Measured on p620 (R9700, IQ3_S, 8K prompt, 24576+ // expert pairs of which 13207 resident): PLE block 845 ms direct / 451 ms mmap / 132 ms resident,+ // prompt 2125 / 2368 / 2611 tok/s, and the answer md5 was identical in all three. The RAM cost is the+ // table itself; the policy needs 8 GiB of headroom on top of it (available_host_bytes accounts for+ // cgroups and --memory-limit-mib), an explicit --ple-io always wins, and STRATA_PLE_AUTO_RAM=0 turns it off.+ bool ple_auto_ram = false;+ if (pio.mode == strata::kernels::PleIo::Direct && !o.ple_io_explicit) {+ const strata::kernels::PleIoOptions direct_pio = pio;+ const char* auto_ram = std::getenv("STRATA_PLE_AUTO_RAM");+ const bool allowed = auto_ram == nullptr || std::atoi(auto_ram) != 0;+ const uint64_t need = ple_table.table_bytes();+ const auto available = strata::core::available_host_bytes();+ if (allowed && need > 0 && strata::core::conversation_memory_admit(available, need, 8ull << 30)) {+ ple_table.close();+ pio.mode = strata::kernels::PleIo::Mmap;+ pio.lock = true;+ pio.keepalive_ms = 0;+ std::string ram_err;+ if (ple_table.open(o.ple_gguf, ram_err, pio)) {+ ple_auto_ram = true;+ std::fprintf(stderr, "strata generate: --ple-io ram chosen by the residency policy (the table is "+ "%.1f GiB and %.1f GiB of RAM is available; STRATA_PLE_AUTO_RAM=0 keeps the "+ "direct reads, --ple-io direct forces them)\n",+ (double) need / (1ull << 30), (double) *available / (1ull << 30));+ } else {+ // Back to what was asked for; a table that cannot be mapped must not stop the engine.+ std::fprintf(stderr, "strata generate: resident PLE table failed (%s); keeping direct reads\n",+ ram_err.c_str());+ pio = direct_pio;+ if (!ple_table.open(o.ple_gguf, err, pio)) {+ std::fprintf(stderr, "strata generate: %s\n", err.c_str());+ return 1;+ }+ }+ }+ }// Recommend, never force: when the default direct reads measure slow, switch to the mapped reads (40-100x// faster on the reporters' drives); an explicit --ple-io direct is kept and only warned about.if (pio.mode == strata::kernels::PleIo::Direct && std::getenv("STRATA_PLE_PROBE") == nullptr) {@@ -4106,8 +4144,9 @@ int main(int argc, char** argv) {}#endifif (pio.lock)- std::fprintf(stderr, "strata generate: PLE table %s (--ple-io ram) in %.1f s\n",+ std::fprintf(stderr, "strata generate: PLE table %s (%s) in %.1f s\n",ple_table.locked() ? "locked in RAM" : "loaded (not locked)",+ ple_auto_ram ? "--ple-io ram chosen by the residency policy" : "--ple-io ram",std::chrono::duration<double>(Clock::now() - tpl).count());if (pio.keepalive_ms > 0)std::fprintf(stderr, "strata generate: the SSD is kept awake while rows are read: one page of the table after "