Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

149: STRATA_IO_PF_GATE=1 - the decode's read-ahead only where the page cache has room and it pays (opt-in)

Niko12218b1139d026c9
diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hpp
index 388358d5f..74cfdd955 100644
--- a/include/strata/core/expert_source.hpp
+++ b/include/strata/core/expert_source.hpp
@@ -252,6 +252,8 @@ public:
return got;
}
virtual void prefetch_end() {}
+ /// 149 STRATA_IO_PF_GATE: whether the decode's read-ahead runs now (the look-ahead then predicts one layer, not two)
+ virtual bool read_ahead_open() const { return true; }
/// 149-cold STRATA_COLD_OVERLAP=3: the last prefetch_begin left this expert unstaged because the page cache holds
/// all of it (the GGUF read in place): its gate, up and down rows in the mapping. False: ask `blob`.
virtual bool zc_roles(int64_t layer, int64_t expert, const uint8_t** p) { (void) layer; (void) expert; (void) p; return false; }
@@ -969,6 +971,42 @@ private:
void claim_todo(int64_t layer, const int64_t* experts, int64_t n, std::vector<Fill>& todo, uint8_t* pending,
bool zc = false);
std::vector<int64_t> zc_keys_; ///< 149-cold mode 3: the keys the last prefetch_begin left in the mapping
+public:
+ /// 149 STRATA_IO_PF_GATE=1 (with STRATA_IO_PREFETCH=1): the decode's read-ahead and its pread demand reads run only
+ /// while the page cache can hold enough of the file tier: (the memory available, cgroup limits included, + the
+ /// file-tier bytes cached now) / the experts' bytes, checked every 16 decode windows; it closes below
+ /// STRATA_IO_PF_GATE_LO 0.55 and opens above STRATA_IO_PF_GATE_HI 0.65. Closed, the decode reads as without io
+ /// prefetch; the prompt path is not gated. (A held share at demand would not do: the read-ahead itself raises it.)
+ /// `DecodeScope` marks the decode thread (and the fetch threads it starts).
+ double gate_ratio() const { return gate_ratio_; }
+ struct DecodeScope {
+ bool prev;
+ DecodeScope();
+ ~DecodeScope();
+ };
+ static bool in_decode();
+ bool gate_open() const { return !gate_on_ || gate_open_.load(std::memory_order_relaxed); }
+ bool read_ahead_open() const override { return gate_open(); }
+ bool gate_on() const { return gate_on_; }
+ double gate_held() const { return gate_ewma_; }
+ int64_t gate_closed_windows() const { return gate_closed_w_; }
+ int64_t gate_flips() const { return gate_flips_; }
+private:
+ void gate_tick();
+ void gate_capacity();
+ bool gate_on_ = false;
+ double gate_lo_ = 0.55, gate_hi_ = 0.65, gate_ewma_ = -1.0, gate_ratio_ = -1.0;
+ int64_t gate_win_ = 0, gate_useless_since_ = 0;
+ uint64_t gate_prev_pf_ = 0, gate_prev_used_ = 0;
+ double gate_useful_ = 1.0, gate_min_useful_ = 0.10;
+ bool gate_useless_ = false;
+public:
+ double gate_useful() const { return gate_useful_; }
+private:
+ std::atomic<bool> gate_open_{true};
+ mutable std::atomic<uint64_t> gate_c_{0}, gate_u_{0};
+ uint64_t gate_prev_c_ = 0, gate_prev_u_ = 0;
+ int64_t gate_closed_w_ = 0, gate_flips_ = 0;
std::vector<Fill> async_todo_;
std::vector<std::thread> async_th_;
std::atomic<size_t> async_next_{0};
diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp
index 3f16b2bc1..15bc367ad 100644
--- a/src/core/expert_source.cpp
+++ b/src/core/expert_source.cpp
@@ -702,7 +702,7 @@ void FileExpertSource::io_window_tick() {
// the queue's fill against the window budget (a busy drive takes only the surer guesses); the window's speculative bytes
// stay within the budget; each job carries the layer clock by which it must have started (else the worker drops it).
void FileExpertSource::warm_scored(int64_t layer, const int64_t* experts, const float* conf, int64_t n, int ahead) {
- if (!io_sched_ || !io_pf_ || !direct_.empty() || !io_ahead_) { warm(layer, experts, n); return; }
+ if (!io_sched_ || !io_pf_ || !direct_.empty() || !io_ahead_ || !gate_open()) { warm(layer, experts, n); return; }
if (n <= 0 || layer < 0 || layer >= n_layers_) return;
const uint64_t bytes = layer_blob_bytes_[(size_t) layer];
const uint64_t deadline = io_epoch_.load(std::memory_order_relaxed) + (uint64_t) std::max(1, ahead);
@@ -771,6 +771,10 @@ void FileExpertSource::set_io_prefetch(bool on, int threads) {
if (const char* m = std::getenv("STRATA_IO_PF_BUDGET_MB")) io_budget_fixed_ = (uint64_t) std::max(0.0, std::atof(m) * 1048576.0);
if (const char* m = std::getenv("STRATA_IO_PF_BUDGET_FRAC")) io_budget_frac_ = std::clamp(std::atof(m), 0.01, 4.0);
if (const char* m = std::getenv("STRATA_IO_PF_TARGET")) io_target_ = std::clamp(std::atof(m), 0.0, 1.0);
+ if (const char* m = std::getenv("STRATA_IO_PF_GATE")) gate_on_ = std::atoi(m) != 0;
+ if (const char* m = std::getenv("STRATA_IO_PF_GATE_LO")) gate_lo_ = std::clamp(std::atof(m), 0.0, 1.0);
+ if (const char* m = std::getenv("STRATA_IO_PF_GATE_HI")) gate_hi_ = std::clamp(std::atof(m), gate_lo_, 1.0);
+ if (const char* m = std::getenv("STRATA_IO_PF_GATE_USEFUL")) gate_min_useful_ = std::clamp(std::atof(m), 0.0, 1.0);
}
if (!pf_mark_) {
pf_mark_.reset(new std::atomic<uint8_t>[(size_t) (n_layers_ * n_expert_)]);
@@ -1364,7 +1368,12 @@ bool FileExpertSource::copy_from_files(int64_t layer, int64_t expert, uint8_t* d
const uint64_t res = resident_in(sp[i].p, sp[i].n);
io_cached_.fetch_add(res, std::memory_order_relaxed);
io_uncached_.fetch_add(sp[i].n - res, std::memory_order_relaxed);
- if (io_pf_ && res < sp[i].n && sp[i].fd >= 0) {
+ const bool dec = in_decode();
+ if (gate_on_ && dec) {
+ gate_c_.fetch_add(res, std::memory_order_relaxed);
+ gate_u_.fetch_add(sp[i].n - res, std::memory_order_relaxed);
+ }
+ if (io_pf_ && res < sp[i].n && sp[i].fd >= 0 && !(dec && !gate_open())) {
const auto t0 = std::chrono::steady_clock::now();
const bool ok = pread_full(sp[i].fd, dst + sp[i].at, sp[i].n, sp[i].off);
io_pread_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),
@@ -1506,6 +1515,110 @@ bool FileExpertSource::claim_stage(int64_t key, size_t& v, bool& fill, bool ahea
return false;
}
+// 149 STRATA_IO_PF_GATE: whether this thread reads for the decode (the dispatch's thread, and the fetch threads it starts)
+namespace { thread_local bool tl_decode = false; }
+FileExpertSource::DecodeScope::DecodeScope() : prev(tl_decode) { tl_decode = true; }
+FileExpertSource::DecodeScope::~DecodeScope() { tl_decode = prev; }
+bool FileExpertSource::in_decode() { return tl_decode; }
+
+// once a decode window: the share of the window's demand bytes the page cache held, an EWMA, and the gate's state
+void FileExpertSource::gate_tick() {
+ if (!gate_on_) return;
+ const uint64_t c = gate_c_.load(std::memory_order_relaxed), u = gate_u_.load(std::memory_order_relaxed);
+ const uint64_t dc = c - gate_prev_c_, du = u - gate_prev_u_;
+ gate_prev_c_ = c;
+ gate_prev_u_ = u;
+ if (dc + du < (64ull << 20)) { // too few bytes read this window to judge; the state stays
+ if (!gate_open_.load(std::memory_order_relaxed)) ++gate_closed_w_;
+ return;
+ }
+ const double h = (double) dc / (double) (dc + du);
+ gate_ewma_ = gate_ewma_ < 0.0 ? h : 0.8 * gate_ewma_ + 0.2 * h; // reported only
+ if (!gate_open_.load(std::memory_order_relaxed)) ++gate_closed_w_;
+}
+
+namespace {
+// The room the page cache has for the experts: in a memory-limited cgroup (v2; the tightest limit of the group and its
+// ancestors) the limit less what is not page cache (the group's memory.current - its "file" pages); otherwise
+// MemAvailable (free + reclaimable cache). 0 when unknown.
+uint64_t gate_cache_room() {
+#if defined(__linux__)
+ auto read_u64 = [](const std::string& path, uint64_t& v) {
+ std::ifstream f(path);
+ std::string s;
+ if (!(f >> s) || s == "max") return false;
+ v = std::strtoull(s.c_str(), nullptr, 10);
+ return v > 0;
+ };
+ std::string rel;
+ {
+ std::ifstream g("/proc/self/cgroup");
+ std::string line;
+ while (std::getline(g, line))
+ if (line.rfind("0::", 0) == 0) { rel = line.substr(3); break; }
+ }
+ uint64_t best = 0;
+ if (!rel.empty()) {
+ const std::string base = "/sys/fs/cgroup";
+ for (std::string at = rel;; ) {
+ const std::string dir = base + (at == "/" ? std::string() : at);
+ uint64_t mx = 0, cur = 0, file = 0;
+ if (read_u64(dir + "/memory.max", mx) && read_u64(dir + "/memory.current", cur)) {
+ std::ifstream st(dir + "/memory.stat");
+ std::string k;
+ uint64_t v = 0;
+ while (st >> k >> v)
+ if (k == "file") { file = v; break; }
+ const uint64_t used = cur > file ? cur - file : 0;
+ const uint64_t room = mx > used ? mx - used : 0;
+ if (best == 0 || room < best) best = room;
+ }
+ if (at.empty() || at == "/") break;
+ const size_t cut = at.find_last_of('/');
+ at = cut == 0 || cut == std::string::npos ? std::string("/") : at.substr(0, cut);
+ }
+ }
+ if (best > 0) return best;
+ std::ifstream info("/proc/meminfo");
+ std::stringstream text;
+ text << info.rdbuf();
+ return detail::meminfo_bytes(text.str(), "MemAvailable");
+#else
+ return 0;
+#endif
+}
+}
+
+void FileExpertSource::gate_capacity() {
+ if (!gate_on_ || (gate_win_++ % 16) != 0) return;
+ const uint64_t total = expert_bytes(), room = gate_cache_room();
+ if (total == 0 || room == 0) return; // unknown: the state stays
+ gate_ratio_ = (double) room / (double) total;
+ // and the read-ahead must pay: the share of its bytes a layer then used, over these 16 windows (64 MB at least read
+ // ahead to judge). Below STRATA_IO_PF_GATE_USEFUL (0.10) it closes (a RAM tier holds what it would read: an 18 GiB
+ // RAM tier, 0% used, -9..-15% decode); closed for that reason, it opens again for a probe every 256 windows.
+ const uint64_t pf = io_pf_bytes_.load(std::memory_order_relaxed), used = io_pf_used_bytes_.load(std::memory_order_relaxed);
+ const uint64_t dpf = pf - gate_prev_pf_, dused = used - gate_prev_used_;
+ gate_prev_pf_ = pf;
+ gate_prev_used_ = used;
+ if (dpf >= (64ull << 20)) gate_useful_ = (double) dused / (double) dpf;
+ const bool open = gate_open_.load(std::memory_order_relaxed);
+ const bool room_ok = open ? gate_ratio_ >= gate_lo_ : gate_ratio_ > gate_hi_;
+ bool want = room_ok;
+ // too little to read ahead (< 128 MB in 16 windows: the RAM and VRAM tiers hold what is predicted) costs the look-ahead's
+ // second layer and the I/O threads for nothing (an 18 GiB RAM tier: 2.5 GB read ahead in a run, -9..-16% decode)
+ if (room_ok && open && gate_win_ > 16 && (dpf < (128ull << 20) || gate_useful_ < gate_min_useful_)) {
+ want = false;
+ gate_useless_ = true;
+ gate_useless_since_ = gate_win_;
+ } else if (room_ok && !open && gate_useless_) {
+ want = gate_win_ - gate_useless_since_ >= 256; // the probe
+ if (want) gate_useless_ = false;
+ }
+ if (!room_ok) gate_useless_ = false;
+ if (want != open) { gate_open_.store(want, std::memory_order_relaxed); ++gate_flips_; }
+}
+
// Fills buffer `v` (reserved by claim_stage) outside the lock, then publishes it.
bool FileExpertSource::fill_stage(size_t v, int64_t layer, int64_t expert, uint8_t* dst) {
const auto t0 = std::chrono::steady_clock::now();
@@ -1723,7 +1836,9 @@ int64_t FileExpertSource::prefetch_begin(int64_t layer, const int64_t* experts,
async_next_.store(0, std::memory_order_relaxed);
async_end_ns_.store(0, std::memory_order_relaxed);
const bool direct = !direct_.empty();
- auto work = [this, direct] {
+ const bool dec = tl_decode;
+ auto work = [this, direct, dec] {
+ tl_decode = dec;
const std::vector<Fill>& todo = async_todo_;
if (direct) {
for (size_t at; (at = async_next_.fetch_add(16)) < todo.size();) {
@@ -1780,6 +1895,7 @@ bool FileExpertSource::zc_roles(int64_t layer, int64_t expert, const uint8_t** p
if (complement_ready_) file_reads_.fetch_add(1, std::memory_order_relaxed);
reads_.fetch_add(1, std::memory_order_relaxed);
if (io_stats_) io_cached_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);
+ if (gate_on_ && tl_decode) gate_c_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);
return true;
}
@@ -1881,7 +1997,9 @@ void FileExpertSource::fill_many(const std::vector<Fill>& todo) {
#endif
// the page faults of a mapped read are one outstanding request each: several threads keep the SSD's queue full
std::atomic<size_t> next{0};
+ const bool dec = tl_decode;
auto work = [&] {
+ tl_decode = dec;
for (size_t i; (i = next.fetch_add(1)) < todo.size();)
(void) fill_stage(todo[i].v, todo[i].layer, todo[i].e, todo[i].dst);
};
@@ -2467,7 +2585,7 @@ void FileExpertSource::warm(int64_t layer, const int64_t* experts, int64_t n) {
continue; // in the RAM copy
if (warm_stamp_) warm_stamp_[index].store(stamp, std::memory_order_relaxed);
warm_count_.fetch_add(1, std::memory_order_relaxed);
- if (io_pf_ && direct_.empty()) { // read it ahead with pread on the I/O threads (instead of an madvise hint)
+ if (io_pf_ && direct_.empty() && gate_open()) { // read it ahead with pread on the I/O threads (instead of an madvise hint)
if (io_ahead_) io_enqueue(layer, e, true);
continue;
}
@@ -2667,7 +2785,8 @@ void RouterLookahead::run() {
}
const auto t0 = std::chrono::steady_clock::now();
const int64_t layer0 = layer;
- for (int dj = 0; dj < depth_ && layer0 + dj < (int64_t) routers_.size(); ++dj) {
+ const int depth = src_ != nullptr && !src_->read_ahead_open() ? 1 : depth_; // 149: the gate closed
+ for (int dj = 0; dj < depth && layer0 + dj < (int64_t) routers_.size(); ++dj) {
layer = layer0 + dj;
want.clear();
wconf.clear();
@@ -2780,6 +2899,7 @@ void FileExpertSource::begin_layer(int64_t layer, const int32_t* ids, int64_t k)
}
}
if (new_window && io_sched_) io_window_tick();
+ if (new_window && gate_on_ && tl_decode) { gate_capacity(); gate_tick(); }
}
bool FileExpertSource::transient(int64_t layer, int64_t expert) const {
@@ -4248,6 +4368,7 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
if (ptrace) { std::fprintf(stderr, "pool trace: layer %lld %s %lld\n", (long long) d.layers, what, a); std::fflush(stderr); }
};
const auto c0 = std::chrono::steady_clock::now();
+ const FileExpertSource::DecodeScope decode_scope; // 149 STRATA_IO_PF_GATE: this thread reads for the decode
// 149-cold STRATA_COLD_OVERLAP: 0 off (default), 1 overlap, 2 measure only, 3 overlap + zero-copy, 4 zero-copy only
static const int cold_env = [] {
const char* v = std::getenv("STRATA_COLD_OVERLAP");
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 06a2281bd..47235e070 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -10759,6 +10759,7 @@ int main(int argc, char** argv) {
int64_t ba_window = 0, bt_adapt_rounds = 0, bt_adapt_swaps = 0;
double bt_adapt_wait = 0;
bool in_prompt_read = false; // batch_step runs between a prompt's chunks: the adaptive tier stands still then
+ double serve_adapt_drain_ms = 0.0; // 149: the request's wait for the adaptive round in flight
int64_t bt_a_rounds0 = 0, bt_a_swapped0 = 0; // the asynchronous tier's counters when the timed windows began
Clock::time_point bt_start = Clock::now();
int admit_slot = -1; ///< the slot the request being read will continue in (BGEN)
@@ -13278,7 +13279,11 @@ int main(int argc, char** argv) {
return true;
};
apply_pending(true);
- if (!adapt_tick(true)) { // --adapt-async: the round in flight lands before the prompt lends any slot
+ const auto drain_t0 = Clock::now();
+ const bool drained_ok = adapt_tick(true);
+ // 149: how long the request waited for the adaptive round still in flight (in the prompt's time)
+ serve_adapt_drain_ms = std::chrono::duration<double, std::milli>(Clock::now() - drain_t0).count();
+ if (!drained_ok) { // --adapt-async: the round in flight lands before the prompt lends any slot
std::printf("ERR an adaptive refill failed\n");
return 1;
}
@@ -14845,6 +14850,9 @@ int main(int argc, char** argv) {
prompt_ms > 0 ? 1000.0 * read_n / prompt_ms : 0.0, (long long) produced_n, decode_ms,
decode_ms > 0 ? 1000.0 * produced_n / decode_ms : 0.0, (long long) draft_accepted,
(long long) draft_offered, checks.size(), cancelled ? " (cancelled)" : "");
+ if (serve_adapt_drain_ms >= 1.0)
+ std::fprintf(stderr, "strata serve: the request first waited %.0f ms for the adaptive tier's round in flight\n",
+ serve_adapt_drain_ms);
// the VRAM share of the experts the pool looked up while decoding; experts it sent over PCIe for the GPU
// to read (--pcie-frac) are in neither count - #588: so that share is said beside it (raising --pcie-frac
// raises the hit rate while the PCIe reads may make the decode slower)
@@ -14925,6 +14933,12 @@ int main(int argc, char** argv) {
(unsigned long long) (ic.pf_dropped - ic_prev.pf_dropped),
(double) (ic.crit_us - ic_prev.crit_us) / 1e3,
(unsigned long long) (ic.crit_n - ic_prev.crit_n));
+ if (src.io_prefetch() && src.gate_on()) // 149 STRATA_IO_PF_GATE: the decode's held share, the gate
+ std::fprintf(stderr, "; decode gate: cache room %.2f of the experts, read-ahead used %.0f%%, page cache "
+ "held %.0f%% of the decode's reads (EWMA), %s, closed in %lld windows so far, %lld flips",
+ src.gate_ratio(), 100.0 * src.gate_useful(), 100.0 * std::max(0.0, src.gate_held()),
+ src.gate_open() ? "open" : "closed", (long long) src.gate_closed_windows(),
+ (long long) src.gate_flips());
if (src.io_prefetch()) // 149-prefetch: bytes read ahead, of them used before their demand; the scheduler's drops
std::fprintf(stderr, "; read ahead %.1f MB, %.1f MB of it used; %llu late, %llu not admitted (confidence), "
"%llu (budget %.0f MB/window)",