Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
149: STRATA_IO_PF_GATE=1 - the decode's read-ahead only where the page cache has room and it pays (opt-in)
Niko12218b1139d026c9
diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hppindex 388358d5f..74cfdd955 100644--- a/include/strata/core/expert_source.hpp+++ b/include/strata/core/expert_source.hpp@@ -252,6 +252,8 @@ public:return got;}virtual void prefetch_end() {}+ /// 149 STRATA_IO_PF_GATE: whether the decode's read-ahead runs now (the look-ahead then predicts one layer, not two)+ virtual bool read_ahead_open() const { return true; }/// 149-cold STRATA_COLD_OVERLAP=3: the last prefetch_begin left this expert unstaged because the page cache holds/// all of it (the GGUF read in place): its gate, up and down rows in the mapping. False: ask `blob`.virtual bool zc_roles(int64_t layer, int64_t expert, const uint8_t** p) { (void) layer; (void) expert; (void) p; return false; }@@ -969,6 +971,42 @@ private:void claim_todo(int64_t layer, const int64_t* experts, int64_t n, std::vector<Fill>& todo, uint8_t* pending,bool zc = false);std::vector<int64_t> zc_keys_; ///< 149-cold mode 3: the keys the last prefetch_begin left in the mapping+public:+ /// 149 STRATA_IO_PF_GATE=1 (with STRATA_IO_PREFETCH=1): the decode's read-ahead and its pread demand reads run only+ /// while the page cache can hold enough of the file tier: (the memory available, cgroup limits included, + the+ /// file-tier bytes cached now) / the experts' bytes, checked every 16 decode windows; it closes below+ /// STRATA_IO_PF_GATE_LO 0.55 and opens above STRATA_IO_PF_GATE_HI 0.65. Closed, the decode reads as without io+ /// prefetch; the prompt path is not gated. (A held share at demand would not do: the read-ahead itself raises it.)+ /// `DecodeScope` marks the decode thread (and the fetch threads it starts).+ double gate_ratio() const { return gate_ratio_; }+ struct DecodeScope {+ bool prev;+ DecodeScope();+ ~DecodeScope();+ };+ static bool in_decode();+ bool gate_open() const { return !gate_on_ || gate_open_.load(std::memory_order_relaxed); }+ bool read_ahead_open() const override { return gate_open(); }+ bool gate_on() const { return gate_on_; }+ double gate_held() const { return gate_ewma_; }+ int64_t gate_closed_windows() const { return gate_closed_w_; }+ int64_t gate_flips() const { return gate_flips_; }+private:+ void gate_tick();+ void gate_capacity();+ bool gate_on_ = false;+ double gate_lo_ = 0.55, gate_hi_ = 0.65, gate_ewma_ = -1.0, gate_ratio_ = -1.0;+ int64_t gate_win_ = 0, gate_useless_since_ = 0;+ uint64_t gate_prev_pf_ = 0, gate_prev_used_ = 0;+ double gate_useful_ = 1.0, gate_min_useful_ = 0.10;+ bool gate_useless_ = false;+public:+ double gate_useful() const { return gate_useful_; }+private:+ std::atomic<bool> gate_open_{true};+ mutable std::atomic<uint64_t> gate_c_{0}, gate_u_{0};+ uint64_t gate_prev_c_ = 0, gate_prev_u_ = 0;+ int64_t gate_closed_w_ = 0, gate_flips_ = 0;std::vector<Fill> async_todo_;std::vector<std::thread> async_th_;std::atomic<size_t> async_next_{0};diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cppindex 3f16b2bc1..15bc367ad 100644--- a/src/core/expert_source.cpp+++ b/src/core/expert_source.cpp@@ -702,7 +702,7 @@ void FileExpertSource::io_window_tick() {// the queue's fill against the window budget (a busy drive takes only the surer guesses); the window's speculative bytes// stay within the budget; each job carries the layer clock by which it must have started (else the worker drops it).void FileExpertSource::warm_scored(int64_t layer, const int64_t* experts, const float* conf, int64_t n, int ahead) {- if (!io_sched_ || !io_pf_ || !direct_.empty() || !io_ahead_) { warm(layer, experts, n); return; }+ if (!io_sched_ || !io_pf_ || !direct_.empty() || !io_ahead_ || !gate_open()) { warm(layer, experts, n); return; }if (n <= 0 || layer < 0 || layer >= n_layers_) return;const uint64_t bytes = layer_blob_bytes_[(size_t) layer];const uint64_t deadline = io_epoch_.load(std::memory_order_relaxed) + (uint64_t) std::max(1, ahead);@@ -771,6 +771,10 @@ void FileExpertSource::set_io_prefetch(bool on, int threads) {if (const char* m = std::getenv("STRATA_IO_PF_BUDGET_MB")) io_budget_fixed_ = (uint64_t) std::max(0.0, std::atof(m) * 1048576.0);if (const char* m = std::getenv("STRATA_IO_PF_BUDGET_FRAC")) io_budget_frac_ = std::clamp(std::atof(m), 0.01, 4.0);if (const char* m = std::getenv("STRATA_IO_PF_TARGET")) io_target_ = std::clamp(std::atof(m), 0.0, 1.0);+ if (const char* m = std::getenv("STRATA_IO_PF_GATE")) gate_on_ = std::atoi(m) != 0;+ if (const char* m = std::getenv("STRATA_IO_PF_GATE_LO")) gate_lo_ = std::clamp(std::atof(m), 0.0, 1.0);+ if (const char* m = std::getenv("STRATA_IO_PF_GATE_HI")) gate_hi_ = std::clamp(std::atof(m), gate_lo_, 1.0);+ if (const char* m = std::getenv("STRATA_IO_PF_GATE_USEFUL")) gate_min_useful_ = std::clamp(std::atof(m), 0.0, 1.0);}if (!pf_mark_) {pf_mark_.reset(new std::atomic<uint8_t>[(size_t) (n_layers_ * n_expert_)]);@@ -1364,7 +1368,12 @@ bool FileExpertSource::copy_from_files(int64_t layer, int64_t expert, uint8_t* dconst uint64_t res = resident_in(sp[i].p, sp[i].n);io_cached_.fetch_add(res, std::memory_order_relaxed);io_uncached_.fetch_add(sp[i].n - res, std::memory_order_relaxed);- if (io_pf_ && res < sp[i].n && sp[i].fd >= 0) {+ const bool dec = in_decode();+ if (gate_on_ && dec) {+ gate_c_.fetch_add(res, std::memory_order_relaxed);+ gate_u_.fetch_add(sp[i].n - res, std::memory_order_relaxed);+ }+ if (io_pf_ && res < sp[i].n && sp[i].fd >= 0 && !(dec && !gate_open())) {const auto t0 = std::chrono::steady_clock::now();const bool ok = pread_full(sp[i].fd, dst + sp[i].at, sp[i].n, sp[i].off);io_pread_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),@@ -1506,6 +1515,110 @@ bool FileExpertSource::claim_stage(int64_t key, size_t& v, bool& fill, bool aheareturn false;}+// 149 STRATA_IO_PF_GATE: whether this thread reads for the decode (the dispatch's thread, and the fetch threads it starts)+namespace { thread_local bool tl_decode = false; }+FileExpertSource::DecodeScope::DecodeScope() : prev(tl_decode) { tl_decode = true; }+FileExpertSource::DecodeScope::~DecodeScope() { tl_decode = prev; }+bool FileExpertSource::in_decode() { return tl_decode; }++// once a decode window: the share of the window's demand bytes the page cache held, an EWMA, and the gate's state+void FileExpertSource::gate_tick() {+ if (!gate_on_) return;+ const uint64_t c = gate_c_.load(std::memory_order_relaxed), u = gate_u_.load(std::memory_order_relaxed);+ const uint64_t dc = c - gate_prev_c_, du = u - gate_prev_u_;+ gate_prev_c_ = c;+ gate_prev_u_ = u;+ if (dc + du < (64ull << 20)) { // too few bytes read this window to judge; the state stays+ if (!gate_open_.load(std::memory_order_relaxed)) ++gate_closed_w_;+ return;+ }+ const double h = (double) dc / (double) (dc + du);+ gate_ewma_ = gate_ewma_ < 0.0 ? h : 0.8 * gate_ewma_ + 0.2 * h; // reported only+ if (!gate_open_.load(std::memory_order_relaxed)) ++gate_closed_w_;+}++namespace {+// The room the page cache has for the experts: in a memory-limited cgroup (v2; the tightest limit of the group and its+// ancestors) the limit less what is not page cache (the group's memory.current - its "file" pages); otherwise+// MemAvailable (free + reclaimable cache). 0 when unknown.+uint64_t gate_cache_room() {+#if defined(__linux__)+ auto read_u64 = [](const std::string& path, uint64_t& v) {+ std::ifstream f(path);+ std::string s;+ if (!(f >> s) || s == "max") return false;+ v = std::strtoull(s.c_str(), nullptr, 10);+ return v > 0;+ };+ std::string rel;+ {+ std::ifstream g("/proc/self/cgroup");+ std::string line;+ while (std::getline(g, line))+ if (line.rfind("0::", 0) == 0) { rel = line.substr(3); break; }+ }+ uint64_t best = 0;+ if (!rel.empty()) {+ const std::string base = "/sys/fs/cgroup";+ for (std::string at = rel;; ) {+ const std::string dir = base + (at == "/" ? std::string() : at);+ uint64_t mx = 0, cur = 0, file = 0;+ if (read_u64(dir + "/memory.max", mx) && read_u64(dir + "/memory.current", cur)) {+ std::ifstream st(dir + "/memory.stat");+ std::string k;+ uint64_t v = 0;+ while (st >> k >> v)+ if (k == "file") { file = v; break; }+ const uint64_t used = cur > file ? cur - file : 0;+ const uint64_t room = mx > used ? mx - used : 0;+ if (best == 0 || room < best) best = room;+ }+ if (at.empty() || at == "/") break;+ const size_t cut = at.find_last_of('/');+ at = cut == 0 || cut == std::string::npos ? std::string("/") : at.substr(0, cut);+ }+ }+ if (best > 0) return best;+ std::ifstream info("/proc/meminfo");+ std::stringstream text;+ text << info.rdbuf();+ return detail::meminfo_bytes(text.str(), "MemAvailable");+#else+ return 0;+#endif+}+}++void FileExpertSource::gate_capacity() {+ if (!gate_on_ || (gate_win_++ % 16) != 0) return;+ const uint64_t total = expert_bytes(), room = gate_cache_room();+ if (total == 0 || room == 0) return; // unknown: the state stays+ gate_ratio_ = (double) room / (double) total;+ // and the read-ahead must pay: the share of its bytes a layer then used, over these 16 windows (64 MB at least read+ // ahead to judge). Below STRATA_IO_PF_GATE_USEFUL (0.10) it closes (a RAM tier holds what it would read: an 18 GiB+ // RAM tier, 0% used, -9..-15% decode); closed for that reason, it opens again for a probe every 256 windows.+ const uint64_t pf = io_pf_bytes_.load(std::memory_order_relaxed), used = io_pf_used_bytes_.load(std::memory_order_relaxed);+ const uint64_t dpf = pf - gate_prev_pf_, dused = used - gate_prev_used_;+ gate_prev_pf_ = pf;+ gate_prev_used_ = used;+ if (dpf >= (64ull << 20)) gate_useful_ = (double) dused / (double) dpf;+ const bool open = gate_open_.load(std::memory_order_relaxed);+ const bool room_ok = open ? gate_ratio_ >= gate_lo_ : gate_ratio_ > gate_hi_;+ bool want = room_ok;+ // too little to read ahead (< 128 MB in 16 windows: the RAM and VRAM tiers hold what is predicted) costs the look-ahead's+ // second layer and the I/O threads for nothing (an 18 GiB RAM tier: 2.5 GB read ahead in a run, -9..-16% decode)+ if (room_ok && open && gate_win_ > 16 && (dpf < (128ull << 20) || gate_useful_ < gate_min_useful_)) {+ want = false;+ gate_useless_ = true;+ gate_useless_since_ = gate_win_;+ } else if (room_ok && !open && gate_useless_) {+ want = gate_win_ - gate_useless_since_ >= 256; // the probe+ if (want) gate_useless_ = false;+ }+ if (!room_ok) gate_useless_ = false;+ if (want != open) { gate_open_.store(want, std::memory_order_relaxed); ++gate_flips_; }+}+// Fills buffer `v` (reserved by claim_stage) outside the lock, then publishes it.bool FileExpertSource::fill_stage(size_t v, int64_t layer, int64_t expert, uint8_t* dst) {const auto t0 = std::chrono::steady_clock::now();@@ -1723,7 +1836,9 @@ int64_t FileExpertSource::prefetch_begin(int64_t layer, const int64_t* experts,async_next_.store(0, std::memory_order_relaxed);async_end_ns_.store(0, std::memory_order_relaxed);const bool direct = !direct_.empty();- auto work = [this, direct] {+ const bool dec = tl_decode;+ auto work = [this, direct, dec] {+ tl_decode = dec;const std::vector<Fill>& todo = async_todo_;if (direct) {for (size_t at; (at = async_next_.fetch_add(16)) < todo.size();) {@@ -1780,6 +1895,7 @@ bool FileExpertSource::zc_roles(int64_t layer, int64_t expert, const uint8_t** pif (complement_ready_) file_reads_.fetch_add(1, std::memory_order_relaxed);reads_.fetch_add(1, std::memory_order_relaxed);if (io_stats_) io_cached_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);+ if (gate_on_ && tl_decode) gate_c_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);return true;}@@ -1881,7 +1997,9 @@ void FileExpertSource::fill_many(const std::vector<Fill>& todo) {#endif// the page faults of a mapped read are one outstanding request each: several threads keep the SSD's queue fullstd::atomic<size_t> next{0};+ const bool dec = tl_decode;auto work = [&] {+ tl_decode = dec;for (size_t i; (i = next.fetch_add(1)) < todo.size();)(void) fill_stage(todo[i].v, todo[i].layer, todo[i].e, todo[i].dst);};@@ -2467,7 +2585,7 @@ void FileExpertSource::warm(int64_t layer, const int64_t* experts, int64_t n) {continue; // in the RAM copyif (warm_stamp_) warm_stamp_[index].store(stamp, std::memory_order_relaxed);warm_count_.fetch_add(1, std::memory_order_relaxed);- if (io_pf_ && direct_.empty()) { // read it ahead with pread on the I/O threads (instead of an madvise hint)+ if (io_pf_ && direct_.empty() && gate_open()) { // read it ahead with pread on the I/O threads (instead of an madvise hint)if (io_ahead_) io_enqueue(layer, e, true);continue;}@@ -2667,7 +2785,8 @@ void RouterLookahead::run() {}const auto t0 = std::chrono::steady_clock::now();const int64_t layer0 = layer;- for (int dj = 0; dj < depth_ && layer0 + dj < (int64_t) routers_.size(); ++dj) {+ const int depth = src_ != nullptr && !src_->read_ahead_open() ? 1 : depth_; // 149: the gate closed+ for (int dj = 0; dj < depth && layer0 + dj < (int64_t) routers_.size(); ++dj) {layer = layer0 + dj;want.clear();wconf.clear();@@ -2780,6 +2899,7 @@ void FileExpertSource::begin_layer(int64_t layer, const int32_t* ids, int64_t k)}}if (new_window && io_sched_) io_window_tick();+ if (new_window && gate_on_ && tl_decode) { gate_capacity(); gate_tick(); }}bool FileExpertSource::transient(int64_t layer, int64_t expert) const {@@ -4248,6 +4368,7 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32if (ptrace) { std::fprintf(stderr, "pool trace: layer %lld %s %lld\n", (long long) d.layers, what, a); std::fflush(stderr); }};const auto c0 = std::chrono::steady_clock::now();+ const FileExpertSource::DecodeScope decode_scope; // 149 STRATA_IO_PF_GATE: this thread reads for the decode// 149-cold STRATA_COLD_OVERLAP: 0 off (default), 1 overlap, 2 measure only, 3 overlap + zero-copy, 4 zero-copy onlystatic const int cold_env = [] {const char* v = std::getenv("STRATA_COLD_OVERLAP");diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 06a2281bd..47235e070 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -10759,6 +10759,7 @@ int main(int argc, char** argv) {int64_t ba_window = 0, bt_adapt_rounds = 0, bt_adapt_swaps = 0;double bt_adapt_wait = 0;bool in_prompt_read = false; // batch_step runs between a prompt's chunks: the adaptive tier stands still then+ double serve_adapt_drain_ms = 0.0; // 149: the request's wait for the adaptive round in flightint64_t bt_a_rounds0 = 0, bt_a_swapped0 = 0; // the asynchronous tier's counters when the timed windows beganClock::time_point bt_start = Clock::now();int admit_slot = -1; ///< the slot the request being read will continue in (BGEN)@@ -13278,7 +13279,11 @@ int main(int argc, char** argv) {return true;};apply_pending(true);- if (!adapt_tick(true)) { // --adapt-async: the round in flight lands before the prompt lends any slot+ const auto drain_t0 = Clock::now();+ const bool drained_ok = adapt_tick(true);+ // 149: how long the request waited for the adaptive round still in flight (in the prompt's time)+ serve_adapt_drain_ms = std::chrono::duration<double, std::milli>(Clock::now() - drain_t0).count();+ if (!drained_ok) { // --adapt-async: the round in flight lands before the prompt lends any slotstd::printf("ERR an adaptive refill failed\n");return 1;}@@ -14845,6 +14850,9 @@ int main(int argc, char** argv) {prompt_ms > 0 ? 1000.0 * read_n / prompt_ms : 0.0, (long long) produced_n, decode_ms,decode_ms > 0 ? 1000.0 * produced_n / decode_ms : 0.0, (long long) draft_accepted,(long long) draft_offered, checks.size(), cancelled ? " (cancelled)" : "");+ if (serve_adapt_drain_ms >= 1.0)+ std::fprintf(stderr, "strata serve: the request first waited %.0f ms for the adaptive tier's round in flight\n",+ serve_adapt_drain_ms);// the VRAM share of the experts the pool looked up while decoding; experts it sent over PCIe for the GPU// to read (--pcie-frac) are in neither count - #588: so that share is said beside it (raising --pcie-frac// raises the hit rate while the PCIe reads may make the decode slower)@@ -14925,6 +14933,12 @@ int main(int argc, char** argv) {(unsigned long long) (ic.pf_dropped - ic_prev.pf_dropped),(double) (ic.crit_us - ic_prev.crit_us) / 1e3,(unsigned long long) (ic.crit_n - ic_prev.crit_n));+ if (src.io_prefetch() && src.gate_on()) // 149 STRATA_IO_PF_GATE: the decode's held share, the gate+ std::fprintf(stderr, "; decode gate: cache room %.2f of the experts, read-ahead used %.0f%%, page cache "+ "held %.0f%% of the decode's reads (EWMA), %s, closed in %lld windows so far, %lld flips",+ src.gate_ratio(), 100.0 * src.gate_useful(), 100.0 * std::max(0.0, src.gate_held()),+ src.gate_open() ? "open" : "closed", (long long) src.gate_closed_windows(),+ (long long) src.gate_flips());if (src.io_prefetch()) // 149-prefetch: bytes read ahead, of them used before their demand; the scheduler's dropsstd::fprintf(stderr, "; read ahead %.1f MB, %.1f MB of it used; %llu late, %llu not admitted (confidence), ""%llu (budget %.0f MB/window)",