Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
149: io prefetch scheduling, RAM_ADAPT2, cold-read overlap and zero-copy GGUF experts (all opt-in)
Niko1221c85cc6dd19d2
diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hppindex 4850b8b6a..388358d5f 100644--- a/include/strata/core/expert_source.hpp+++ b/include/strata/core/expert_source.hpp@@ -30,6 +30,7 @@#include <algorithm>#include <cmath>#include <atomic>+#include <chrono>#include <cstdint>#include <cstddef>#include <condition_variable>@@ -233,9 +234,37 @@ public:/// The `n` experts of `layer` the CPU is about to ask `blob` for, all at once: a source that reads a file may/// fetch them in parallel. The bytes `blob` then returns are the same. Default: nothing.virtual void prefetch(int64_t layer, const int64_t* experts, int64_t n) { (void) layer; (void) experts; (void) n; }+ /// 149-cold (STRATA_COLD_OVERLAP): `prefetch`, but the blobs that must be read are read in the background.+ /// `pending[i]` is set to 1 for each expert whose blob is not ready yet (0: `blob` returns it at once); returns how+ /// many. `wait_landed` then waits until at least one pending expert's blob is ready (its state goes 1 -> 2) and+ /// returns how many landed; `prefetch_end` joins the reads (every claimed blob is published by then). `blob` on+ /// a pending expert still works: it waits for that blob. Default: the synchronous `prefetch`, nothing pending.+ virtual int64_t prefetch_begin(int64_t layer, const int64_t* experts, int64_t n, uint8_t* pending) {+ prefetch(layer, experts, n);+ for (int64_t i = 0; i < n; ++i) pending[i] = 0;+ return 0;+ }+ virtual int64_t wait_landed(int64_t layer, const int64_t* experts, int64_t n, uint8_t* state) {+ (void) layer; (void) experts;+ int64_t got = 0;+ for (int64_t i = 0; i < n; ++i)+ if (state[i] == 1) { state[i] = 2; ++got; }+ return got;+ }+ virtual void prefetch_end() {}+ /// 149-cold STRATA_COLD_OVERLAP=3: the last prefetch_begin left this expert unstaged because the page cache holds+ /// all of it (the GGUF read in place): its gate, up and down rows in the mapping. False: ask `blob`.+ virtual bool zc_roles(int64_t layer, int64_t expert, const uint8_t** p) { (void) layer; (void) expert; (void) p; return false; }/// CS-T: the experts of `layer` a predictor expects next - a source that reads a file may start reading their/// pages now, in the background. Only warms: what `blob` returns is unchanged. Default: nothing.virtual void warm(int64_t layer, const int64_t* experts, int64_t n) { (void) layer; (void) experts; (void) n; }+ /// 149-prefetch (STRATA_IO_PF_SCHED=1): `warm` with each expert's confidence (0..1) and how many layers ahead of the+ /// decode its layer is (the deadline: past it the read is dropped). Default: plain `warm`.+ virtual void warm_scored(int64_t layer, const int64_t* experts, const float* conf, int64_t n, int ahead) {+ (void) conf; (void) ahead; warm(layer, experts, n);+ }+ /// Whether the predictor should compute the confidences for warm_scored (else it calls plain warm).+ virtual bool wants_scores() const { return false; }/// Whether `warm` does anything (the predictor is not run otherwise).virtual bool warms() const { return false; }protected:@@ -564,6 +593,19 @@ struct ExpertDispatch {int64_t tail_skipped = 0;int64_t tail_experts = 0;double ms_plan = 0, ms_actq = 0, ms_jobs = 0, ms_run = 0; ///< verify-window dispatch sections+ /// 149-cold STRATA_COLD_OVERLAP: 0 off (the CPU starts once every blob of the layer is read), 1 the blobs that+ /// must be read are read in the background while the pool computes the ready experts, each landed one computed+ /// as it lands; 2 measure only (read everything first, then the ready experts, then the read ones, timed apart).+ /// The per-layer counters: layers with a blob to read, those experts, the ready ones beside them; R = the reads'+ /// wall time, C_r / C_c = the pool on the ready / the read experts, wait = what the decode thread waited for+ /// reads, overlap_max = the sum over layers of min(R, C_r) (mode 2: the stall an overlap could hide).+ /// 3 = 1 + zero-copy: an expert the page cache holds whole (the GGUF in place, Linux mincore) is computed from the+ /// mapping instead of a stage buffer; 4 = zero-copy with the reads as in mode 0. Same bytes, same bits.+ int cold_mode = 0;+ std::vector<strata::kernels::cpu::NativeFmt> zc_fmt; ///< mode 3: the per-job layouts of the zero-copy experts+ int64_t cold_zc = 0; ///< mode 3: experts computed from the mapping directly+ int64_t cold_layers = 0, cold_pending = 0, cold_ready = 0, cold_batches = 0;+ double ms_cold_r = 0, ms_cold_cr = 0, ms_cold_cc = 0, ms_cold_wait = 0, ms_cold_hide = 0;/// Plan v0.3 P6: decayed routing counts per (layer, expert) during decode (sized by the caller; empty = off),/// which the driver uses to swap the most-routed missing experts into the VRAM tier between rounds.std::vector<float> usage;@@ -756,6 +798,22 @@ public:/// STRATA_EXCHANGE_ROTATE (false, nothing changes).bool replace_resident(int64_t layer, int64_t in, int64_t out);int64_t ram_adapts() const { return ram_adapts_; }+ // ---- 149-prefetch (STRATA_RAM_ADAPT2): the same promotion, staged. `ram_stage` queues the read of `in` on an I/O thread+ // (false: already queued, the queue is full, or no compact copy); `ram_install` - called by the decode thread at a window+ // boundary, when no window reads the RAM copy - puts every read that has landed into `out`'s place, unless the GPU tier+ // took `in` or `out` meanwhile (`host_res`: the residency table, n_expert per layer) or the copy no longer fits.+ bool ram_stage(int64_t layer, int64_t in, int64_t out);+ int64_t ram_install(const int32_t* host_res, int64_t n_expert);+ bool ram_queued(int64_t layer, int64_t expert) const; ///< `expert` is the `in` or `out` of a staged promotion+ int64_t ram_staged_reads() const { return ram_staged_reads_.load(std::memory_order_relaxed); }+ int64_t ram_install_skips() const { return ram_install_skips_.load(std::memory_order_relaxed); }+ int64_t ram_install_gpu() const { return ram_install_gpu_.load(std::memory_order_relaxed); } ///< skipped: the GPU tier took in or out+ /// Where an expert outside the GPU is now: 0 the RAM copy, 1 the files with its pages in the page cache, 2 the files, cold+ /// (Linux: four sampled pages; elsewhere 2).+ int tier_of(int64_t layer, int64_t expert) const;+ uint64_t blob_bytes(int64_t layer) const {+ return layer >= 0 && (size_t) layer < layer_blob_bytes_.size() ? layer_blob_bytes_[(size_t) layer] : 0;+ }bool exchange_rotation() const { return exchange_storage_.active(); }uint64_t rotated_exchanges() const { return exchange_storage_.exchanges(); }uint64_t avoided_exchange_copy_bytes() const { return exchange_storage_.avoided_bytes(); }@@ -832,9 +890,20 @@ public:void begin_layer(int64_t layer, const int32_t* ids, int64_t k) override;/// CS-T: the GGUF in place assembles the missed experts on `fetch_threads_` threads.void prefetch(int64_t layer, const int64_t* experts, int64_t n) override;+ int64_t prefetch_begin(int64_t layer, const int64_t* experts, int64_t n, uint8_t* pending) override;+ int64_t wait_landed(int64_t layer, const int64_t* experts, int64_t n, uint8_t* state) override;+ void prefetch_end() override;+ bool zc_roles(int64_t layer, int64_t expert, const uint8_t** p) override;+ /// 149-cold: the wall time of the last background read batch (prefetch_begin -> its last blob published), us+ double async_fill_us() const { return async_fill_us_; }/// CS-T: the GGUF in place asks the OS for the predicted experts' pages (PrefetchVirtualMemory on Windows,/// madvise(WILLNEED) elsewhere), skipping the RAM copy's.void warm(int64_t layer, const int64_t* experts, int64_t n) override;+ /// 149-prefetch: the scheduled read-ahead (STRATA_IO_PF_SCHED=1 with STRATA_IO_PREFETCH=1): admission by confidence+ /// against the queue's pressure, a speculative byte budget per decode window, a deadline per read.+ void warm_scored(int64_t layer, const int64_t* experts, const float* conf, int64_t n, int ahead) override;+ bool io_sched() const { return io_sched_; }+ bool wants_scores() const override { return io_sched_ && io_pf_; }/// Not when the reads are unbuffered: the warmed pages would be read through the file cache, a second time.bool warms() const override { return (!role_ptr_.empty() || io_pf_) && direct_.empty(); }/// Of the blobs the file tier read for the decode, how many had been warmed for their layer beforehand.@@ -874,6 +943,12 @@ public:uint64_t pf_used = 0, pf_unused = 0; ///< prefetched blobs a layer then used / never useduint64_t crit_us = 0; ///< wall time the decode waited on file reads (prefetch + staged_blob)uint64_t crit_n = 0;+ // 149-prefetch: the read-ahead's bytes and what came of them+ uint64_t pf_bytes = 0; ///< bytes the read-ahead workers read (all modes)+ uint64_t pf_used_bytes = 0; ///< of them, blobs a layer then asked for (arrived before demand)+ uint64_t pf_late = 0; ///< reads dropped because the decode had reached their layer+ uint64_t pf_rej_conf = 0, pf_rej_budget = 0; ///< predictions not admitted (STRATA_IO_PF_SCHED)+ uint64_t pf_budget_bytes = 0; ///< the current per-window speculative budget (scheduler)};IoCounters io_counters() const;@@ -890,6 +965,16 @@ private:struct Fill { size_t v; int64_t layer, e; uint8_t* dst; };/// The claimed buffers' blobs: one overlapped batch when unbuffered, else the fetch threads' mapped copies.void fill_many(const std::vector<Fill>& todo);+ /// 149-cold: the claim half of `prefetch` (the blobs this call must fill; pending[i] = 1 for those, if given)+ void claim_todo(int64_t layer, const int64_t* experts, int64_t n, std::vector<Fill>& todo, uint8_t* pending,+ bool zc = false);+ std::vector<int64_t> zc_keys_; ///< 149-cold mode 3: the keys the last prefetch_begin left in the mapping+ std::vector<Fill> async_todo_;+ std::vector<std::thread> async_th_;+ std::atomic<size_t> async_next_{0};+ std::atomic<int64_t> async_end_ns_{0};+ std::chrono::steady_clock::time_point async_t0_{};+ double async_fill_us_ = 0;/// #286: the blobs from the drive, unbuffered: every role's 4 KiB-aligned window is read at once (overlapped on/// Windows, io_submit on Linux) into this thread's aligned buffer, then copied into place. False when a read fails.bool read_direct(const Fill* fills, size_t n) const;@@ -947,6 +1032,17 @@ private:uint64_t stage_seq_ = 0;int64_t ram_adapts_ = 0; ///< replace_resident: experts moved from the file into the compact copystd::vector<uint8_t> ram_adapt_buf_; ///< ...one blob, read here before it replaces another+ // 149-prefetch: the staged promotions (ram_stage / ram_install)+ struct RamJob { int64_t layer, in, out; std::vector<uint8_t> buf; int state; }; ///< state 0 queued, 1 reading, 2 landed, 3 failed+ mutable std::mutex ram_mu_;+ std::condition_variable ram_cv_;+ std::deque<std::unique_ptr<RamJob>> ram_jobs_;+ std::thread ram_thread_;+ bool ram_quit_ = false;+ std::atomic<int64_t> ram_staged_reads_{0}, ram_install_skips_{0}, ram_install_gpu_{0};+ bool install_resident(int64_t layer, int64_t in, int64_t out, const uint8_t* src); ///< the second half of replace_resident+ void ram_worker();+ void ram_stop();uint64_t epoch_ = 0;int64_t last_layer_ = -1;bool stage_grew_ = false;@@ -971,6 +1067,27 @@ private:std::condition_variable io_cv_;std::deque<std::pair<int32_t, int32_t>> io_q_; ///< (layer, expert) to read aheadbool io_quit_ = false;+ // ---- 149-prefetch: the scheduled read-ahead (STRATA_IO_PF_SCHED=1)+ struct IoJob { int32_t layer, expert; float conf; uint64_t deadline; uint64_t bytes; };+ bool io_sched_ = false;+ float io_minconf_ = 0.5f; ///< STRATA_IO_PF_MINCONF: the bar with an empty queue+ uint64_t io_budget_fixed_ = 0; ///< STRATA_IO_PF_BUDGET_MB (0: auto from the measured rates)+ double io_budget_frac_ = 0.5; ///< STRATA_IO_PF_BUDGET_FRAC: share of what the drive can read in a window+ std::vector<IoJob> io_jobs_; ///< (under io_mu_) admitted, not yet started+ uint64_t io_q_bytes_ = 0; ///< (under io_mu_) their bytes+ uint64_t io_win_bytes_ = 0; ///< (under io_mu_) admitted in the current window+ double io_bw_ = 0.0; ///< (under io_mu_) EWMA bytes/us of one worker's reads+ double io_win_us_ = 0.0; ///< (under io_mu_) EWMA of a decode window's wall time+ std::chrono::steady_clock::time_point io_win_t0_{};+ double io_scale_ = 1.0; ///< (under io_mu_) the budget's feedback factor: shrinks while read-ahead goes unused+ double io_useful_ = -1.0; ///< (under io_mu_) EWMA of used / read-ahead bytes+ uint64_t io_prev_pf_ = 0, io_prev_used_ = 0; ///< (under io_mu_) the counters at the last window tick+ double io_target_ = 0.5; ///< STRATA_IO_PF_TARGET: the used share the feedback aims at (0: no feedback)+ std::atomic<uint64_t> io_epoch_{0}; ///< the decode's layer clock (begin_layer), read by the workers+ std::unique_ptr<std::atomic<uint8_t>[]> pf_mark_; ///< per (layer, expert): read ahead, not yet asked for (fill mode)+ std::atomic<uint64_t> io_pf_bytes_{0}, io_pf_used_bytes_{0}, io_pf_late_{0}, io_pf_rej_conf_{0}, io_pf_rej_budget_{0};+ uint64_t io_budget_now() const; ///< (io_mu_ held) the per-window budget in bytes+ void io_window_tick(); ///< begin_layer: a new decode window starts at layer 0std::vector<char> stage_pf_; ///< per stage buffer: filled ahead, not yet used by a layerstd::vector<int> io_fds_; ///< per mapped file: a plain descriptor (POSIX_FADV_RANDOM) for the preadsmutable std::atomic<uint64_t> io_cached_{0}, io_uncached_{0}, io_pread_bytes_{0}, io_pread_us_{0}, io_pread_n_{0};@@ -983,7 +1100,8 @@ private:/// `exact` false: four sampled pages (a blob is cached when they are; a handful of syscalls, not hundreds of pages' walk).uint64_t cached_bytes(int64_t layer, int64_t expert, uint64_t& total, bool exact = true) const;void io_worker();- void io_enqueue(int64_t layer, int64_t expert);+ void io_enqueue(int64_t layer, int64_t expert, bool ahead = false); ///< ahead: a prediction (warm), not a layer's demand+ static constexpr int32_t kIoAhead = 1 << 30; ///< io_q_: the expert field's flag for a predictionvoid io_stop();const uint8_t* io_blob(int64_t layer, int64_t expert);const uint8_t* base_ = nullptr;diff --git a/include/strata/kernels/cpu/pool.hpp b/include/strata/kernels/cpu/pool.hppindex 686abd6e6..bf0a9bc04 100644--- a/include/strata/kernels/cpu/pool.hpp+++ b/include/strata/kernels/cpu/pool.hpp@@ -61,6 +61,9 @@ struct ExpertJobMulti {float* out[MAXT] = {};/// Plan v0.3 P6: a native pack's activations (the layer's `vec_dot_type`), one per token.const void* nact[MAXT] = {};+ /// 149-cold: this job's own layout (null: the layer's) - the GGUF read in place, where gate / up / down are three+ /// tensors: `blob` is the gate rows and the offsets reach the expert's up and down rows in the mapping.+ const NativeFmt* fmt = nullptr;};/// How worker threads are allocated across physical/logical CPU cores (#272). `All` is the layout the pool hasdiff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hppindex 6c4260824..a34ee6fb5 100644--- a/include/strata/prefill/prefill.hpp+++ b/include/strata/prefill/prefill.hpp@@ -192,6 +192,9 @@ public:/// says how many of the run's tokens that is (-1 when the run read them all). A read beside the batch slots/// (STRATA_PIPED_CONCURRENT) gives way to a waiting shorter request there.std::function<bool()> stop_at_chunk;+ /// 149-prefetch (STRATA_RAM_ADAPT2): a layer's routed expert ids (T x k, -1 = none) after the prompt path downloaded them+ /// for its grouping on the host; a policy may count them. Not called on the paths that keep the ids on the GPU.+ std::function<void(int64_t layer, const int32_t* ids, int64_t n)> on_routes;int64_t stopped_at() const { return stopped_at_; }/// Grows this stage's (and the later stages') pinned upload slots to what a text chunk of `T` tokens uploads, so/// a run of such chunks allocates no pinned memory (STRATA_PIPED_CONCURRENT: an allocation or free on the read'sdiff --git a/src/core/expert_source.cpp b/src/core/expert_source.cppindex d3fefeeff..a97626a66 100644--- a/src/core/expert_source.cpp+++ b/src/core/expert_source.cpp@@ -653,9 +653,87 @@ FileExpertSource::IoCounters FileExpertSource::io_counters() const {c.pf_unused = io_pf_unused_.load(std::memory_order_relaxed);c.crit_us = io_crit_us_.load(std::memory_order_relaxed);c.crit_n = io_crit_n_.load(std::memory_order_relaxed);+ c.pf_bytes = io_pf_bytes_.load(std::memory_order_relaxed);+ c.pf_used_bytes = io_pf_used_bytes_.load(std::memory_order_relaxed);+ c.pf_late = io_pf_late_.load(std::memory_order_relaxed);+ c.pf_rej_conf = io_pf_rej_conf_.load(std::memory_order_relaxed);+ c.pf_rej_budget = io_pf_rej_budget_.load(std::memory_order_relaxed);+ if (io_sched_) {+ std::lock_guard<std::mutex> lk(const_cast<std::mutex&>(io_mu_));+ c.pf_budget_bytes = io_budget_now();+ }return c;}+// 149-prefetch: the speculative bytes a decode window may read ahead. Fixed (STRATA_IO_PF_BUDGET_MB), or what the workers+// measured they can read in one window (one worker's rate x the workers x the window's time) times STRATA_IO_PF_BUDGET_FRAC;+// 256 MB until both rates are known.+uint64_t FileExpertSource::io_budget_now() const {+ if (io_budget_fixed_ > 0) return io_budget_fixed_;+ if (io_bw_ <= 0.0 || io_win_us_ <= 0.0) return 256ull << 20;+ const double b = io_bw_ * (double) std::max(1, io_threads_) * io_win_us_ * io_budget_frac_ * io_scale_;+ return (uint64_t) std::max(16.0 * 1048576.0, b);+}++void FileExpertSource::io_window_tick() {+ const auto now = std::chrono::steady_clock::now();+ std::lock_guard<std::mutex> lk(io_mu_);+ if (io_win_t0_.time_since_epoch().count() != 0) {+ const double us = std::chrono::duration<double, std::micro>(now - io_win_t0_).count();+ io_win_us_ = io_win_us_ <= 0.0 ? us : 0.8 * io_win_us_ + 0.2 * us;+ }+ io_win_t0_ = now;+ io_win_bytes_ = 0;+ // feedback: the share of the read-ahead bytes that a layer then used (the used count lags the reads by a window or+ // so; an EWMA over windows evens that out). Below STRATA_IO_PF_TARGET the budget shrinks (x0.7 a window, to 1/20),+ // above it it grows back (x1.1, to 1): a box whose page cache cannot hold what is read ahead stops paying for it.+ const uint64_t pf = io_pf_bytes_.load(std::memory_order_relaxed), used = io_pf_used_bytes_.load(std::memory_order_relaxed);+ const uint64_t dpf = pf - io_prev_pf_, du = used - io_prev_used_;+ io_prev_pf_ = pf;+ io_prev_used_ = used;+ if (io_target_ > 0.0 && dpf > 0) {+ const double u = std::min(1.0, (double) du / (double) dpf);+ io_useful_ = io_useful_ < 0.0 ? u : 0.8 * io_useful_ + 0.2 * u;+ io_scale_ = io_useful_ < io_target_ ? std::max(0.05, io_scale_ * 0.7) : std::min(1.0, io_scale_ * 1.1);+ }+}++// 149-prefetch: admit the predicted experts that are not resident. An expert's bar is the base confidence, raised with+// the queue's fill against the window budget (a busy drive takes only the surer guesses); the window's speculative bytes+// stay within the budget; each job carries the layer clock by which it must have started (else the worker drops it).+void FileExpertSource::warm_scored(int64_t layer, const int64_t* experts, const float* conf, int64_t n, int ahead) {+ if (!io_sched_ || !io_pf_ || !direct_.empty() || !io_ahead_) { warm(layer, experts, n); return; }+ if (n <= 0 || layer < 0 || layer >= n_layers_) return;+ const uint64_t bytes = layer_blob_bytes_[(size_t) layer];+ const uint64_t deadline = io_epoch_.load(std::memory_order_relaxed) + (uint64_t) std::max(1, ahead);+ bool any = false;+ {+ std::lock_guard<std::mutex> lk(io_mu_);+ const uint64_t budget = io_budget_now();+ for (int64_t j = 0; j < n; ++j) {+ const int64_t e = experts[j];+ if (e < 0 || e >= n_expert_) continue;+ const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) e;+ if (complement_ready_ && resident_blob(index) != nullptr) continue; // in the RAM copy+ warm_count_.fetch_add(1, std::memory_order_relaxed);+ const double pressure = std::min(1.0, (double) io_q_bytes_ / (double) std::max<uint64_t>(1, budget));+ const double bar = (double) io_minconf_ + (1.0 - (double) io_minconf_) * pressure;+ const float c = conf != nullptr ? conf[j] : 1.0f;+ if ((double) c < bar) { io_pf_rej_conf_.fetch_add(1, std::memory_order_relaxed); continue; }+ if (io_win_bytes_ + bytes > budget) { io_pf_rej_budget_.fetch_add(1, std::memory_order_relaxed); continue; }+ bool dup = false;+ for (const IoJob& q : io_jobs_)+ if (q.layer == (int32_t) layer && q.expert == (int32_t) e) { dup = true; break; }+ if (dup) continue;+ io_jobs_.push_back({(int32_t) layer, (int32_t) e, c, deadline, bytes});+ io_q_bytes_ += bytes;+ io_win_bytes_ += bytes;+ any = true;+ }+ }+ if (any) io_cv_.notify_all();+}+void FileExpertSource::set_io_prefetch(bool on, int threads) {io_stop();#if defined(__linux__)@@ -686,6 +764,18 @@ void FileExpertSource::set_io_prefetch(bool on, int threads) {io_pf_ = true;io_stats_ = true;io_quit_ = false;+ { // 149-prefetch: STRATA_IO_PF_SCHED=1 - deadlines, confidence admission and a per-window byte budget (off: as before)+ const char* v = std::getenv("STRATA_IO_PF_SCHED");+ io_sched_ = v != nullptr && std::atoi(v) != 0;+ if (const char* m = std::getenv("STRATA_IO_PF_MINCONF")) io_minconf_ = (float) std::clamp(std::atof(m), 0.0, 1.0);+ if (const char* m = std::getenv("STRATA_IO_PF_BUDGET_MB")) io_budget_fixed_ = (uint64_t) std::max(0.0, std::atof(m) * 1048576.0);+ if (const char* m = std::getenv("STRATA_IO_PF_BUDGET_FRAC")) io_budget_frac_ = std::clamp(std::atof(m), 0.01, 4.0);+ if (const char* m = std::getenv("STRATA_IO_PF_TARGET")) io_target_ = std::clamp(std::atof(m), 0.0, 1.0);+ }+ if (!pf_mark_) {+ pf_mark_.reset(new std::atomic<uint8_t>[(size_t) (n_layers_ * n_expert_)]);+ for (int64_t i = 0; i < n_layers_ * n_expert_; ++i) pf_mark_[(size_t) i].store(0, std::memory_order_relaxed);+ }for (int i = 0; i < io_threads_; ++i) io_workers_.emplace_back([this] { io_worker(); });#else(void) on; (void) threads;@@ -697,6 +787,8 @@ void FileExpertSource::io_stop() {std::lock_guard<std::mutex> lk(io_mu_);io_quit_ = true;io_q_.clear();+ io_jobs_.clear();+ io_q_bytes_ = 0;}io_cv_.notify_all();for (std::thread& t : io_workers_)@@ -710,14 +802,14 @@ void FileExpertSource::io_stop() {io_quit_ = false;}-void FileExpertSource::io_enqueue(int64_t layer, int64_t expert) {+void FileExpertSource::io_enqueue(int64_t layer, int64_t expert, bool ahead) {{std::lock_guard<std::mutex> lk(io_mu_);if (io_q_.size() >= 512) { // the predictions run ahead of the reads: the oldest are the stalestio_q_.pop_front();io_pf_dropped_.fetch_add(1, std::memory_order_relaxed);}- io_q_.emplace_back((int32_t) layer, (int32_t) expert);+ io_q_.emplace_back((int32_t) layer, (int32_t) expert | (ahead ? kIoAhead : 0));}io_cv_.notify_one();}@@ -727,24 +819,66 @@ void FileExpertSource::io_enqueue(int64_t layer, int64_t expert) {void FileExpertSource::io_worker() {for (;;) {int32_t layer, expert;+ bool ahead = true; // a prediction (marked for the used / unused count), not a layer's own demand read{std::unique_lock<std::mutex> lk(io_mu_);- io_cv_.wait(lk, [&] { return io_quit_ || !io_q_.empty(); });+ io_cv_.wait(lk, [&] { return io_quit_ || !io_q_.empty() || !io_jobs_.empty(); });if (io_quit_) return;- layer = io_q_.back().first; // the newest prediction first: it is the layer that comes next- expert = io_q_.back().second;- io_q_.pop_back();+ if (io_sched_ && io_q_.empty() && !io_jobs_.empty()) { // (the layers' own reads in io_q_ go first)+ // 149-prefetch: drop the jobs whose layer the decode has reached (its demand read covers them); then the+ // earliest deadline, the surest guess first among equals+ const uint64_t now = io_epoch_.load(std::memory_order_relaxed);+ size_t best = io_jobs_.size();+ for (size_t i = 0; i < io_jobs_.size();) {+ if (io_jobs_[i].deadline <= now) {+ io_q_bytes_ -= std::min(io_q_bytes_, io_jobs_[i].bytes);+ io_jobs_[i] = io_jobs_.back();+ io_jobs_.pop_back();+ io_pf_late_.fetch_add(1, std::memory_order_relaxed);+ continue;+ }+ ++i;+ }+ best = io_jobs_.size();+ for (size_t i = 0; i < io_jobs_.size(); ++i)+ if (best == io_jobs_.size() || io_jobs_[i].deadline < io_jobs_[best].deadline ||+ (io_jobs_[i].deadline == io_jobs_[best].deadline && io_jobs_[i].conf > io_jobs_[best].conf))+ best = i;+ if (best == io_jobs_.size()) continue; // all were late+ layer = io_jobs_[best].layer;+ expert = io_jobs_[best].expert;+ io_q_bytes_ -= std::min(io_q_bytes_, io_jobs_[best].bytes);+ io_jobs_[best] = io_jobs_.back();+ io_jobs_.pop_back();+ } else {+ if (io_q_.empty()) continue;+ layer = io_q_.back().first; // the newest prediction first: it is the layer that comes next+ expert = io_q_.back().second & ~kIoAhead;+ ahead = (io_q_.back().second & kIoAhead) != 0;+ io_q_.pop_back();+ }}io_pf_jobs_.fetch_add(1, std::memory_order_relaxed);const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;if (complement_ready_ && resident_blob(index) != nullptr) continue;if (!override_.empty() && override_[index] != nullptr) continue;+ const auto tjob = std::chrono::steady_clock::now();+ // 149-prefetch: one worker's read rate (bytes per microsecond), for the scheduler's window budget+ auto note_rate = [&](uint64_t b) {+ if (b == 0) return;+ const double us = std::max(1.0, std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - tjob).count());+ io_pf_bytes_.fetch_add(b, std::memory_order_relaxed);+ std::lock_guard<std::mutex> lk(io_mu_);+ const double r = (double) b / us;+ io_bw_ = io_bw_ <= 0.0 ? r : 0.9 * io_bw_ + 0.1 * r;+ };if (io_fill_) { // read what is not cached into a scratch buffer: the pages stay in the page cache for the kernels#if defined(__linux__)static thread_local std::vector<uint8_t> scratch;Span sp[3];const int ns = spans(layer, expert, sp);bool any = false;+ uint64_t got = 0;for (int i = 0; i < ns; ++i) {if (sp[i].fd < 0 || resident_sampled_n(sp[i].p, sp[i].n)) continue;if (scratch.size() < sp[i].n) scratch.resize((size_t) sp[i].n);@@ -755,9 +889,14 @@ void FileExpertSource::io_worker() {std::memory_order_relaxed);io_pread_n_.fetch_add(1, std::memory_order_relaxed);any = true;+ got += sp[i].n;}}- if (any) io_pf_blobs_.fetch_add(1, std::memory_order_relaxed);+ if (any) {+ io_pf_blobs_.fetch_add(1, std::memory_order_relaxed);+ if (pf_mark_ && ahead) pf_mark_[index].store(1, std::memory_order_relaxed); // read ahead: scored at the demand+ note_rate(got);+ }else io_pf_skips_.fetch_add(1, std::memory_order_relaxed);#endifcontinue;@@ -781,6 +920,7 @@ void FileExpertSource::io_worker() {const bool ok = copy_from_files(layer, expert, dst);io_pf_blobs_.fetch_add(1, std::memory_order_relaxed);if (ok) file_read_bytes_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);+ if (ok) note_rate(layer_blob_bytes_[(size_t) layer]);{std::lock_guard<std::mutex> lk(stage_mu_);stage_busy_[v] = 0;@@ -998,6 +1138,8 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64}void FileExpertSource::close() {+ prefetch_end();+ ram_stop();io_stop();inputs_.clear();if (complement_arena_ != nullptr) {@@ -1281,7 +1423,11 @@ bool FileExpertSource::claim_stage(int64_t key, size_t& v, bool& fill, bool aheav = it->second;stage_epoch_[v] = epoch_;stage_used_[v] = seq;- if (!ahead && stage_pf_[v]) { stage_pf_[v] = 0; io_pf_used_.fetch_add(1, std::memory_order_relaxed); } // a layer asked for what was read ahead+ if (!ahead && stage_pf_[v]) { // a layer asked for what was read ahead+ stage_pf_[v] = 0;+ io_pf_used_.fetch_add(1, std::memory_order_relaxed);+ io_pf_used_bytes_.fetch_add(layer_blob_bytes_[(size_t) (key / n_expert_)], std::memory_order_relaxed);+ }return true;}v = stage_buf_.size();@@ -1455,6 +1601,10 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t nconst int64_t e = experts[i];if (e < 0 || e >= n_expert_) continue;const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) e;+ if (pf_mark_ && pf_mark_[index].exchange(0, std::memory_order_relaxed)) { // 149-prefetch: read ahead, now asked for+ io_pf_used_.fetch_add(1, std::memory_order_relaxed);+ io_pf_used_bytes_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);+ }if (complement_ready_ && resident_blob(index) != nullptr) continue;if (!override_.empty() && override_[index] != nullptr) continue;uint64_t total = 0;@@ -1465,6 +1615,33 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t nreturn;}std::vector<Fill> todo;+ const auto t0 = std::chrono::steady_clock::now();+ claim_todo(layer, experts, n, todo, nullptr);+ fill_many(todo);+ if (io_pf_ && !todo.empty()) {+ io_crit_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),+ std::memory_order_relaxed);+ io_crit_n_.fetch_add(1, std::memory_order_relaxed);+ }+}++void FileExpertSource::claim_todo(int64_t layer, const int64_t* experts, int64_t n, std::vector<Fill>& todo, uint8_t* pending,+ bool zc) {+ // 149-cold mode 3: the GGUF in place, mapped and buffered: an expert the page cache holds whole is computed from+ // the mapping (its three tensors' rows), not assembled into a stage buffer first. Decided here, before the lock.+ zc_keys_.clear();+ std::vector<char> zc_ok;+ if (zc && !role_ptr_.empty() && !unmapped_ && direct_.empty() && strata::kernels::cpu::expert_layout().native &&+ strata::kernels::cpu::expert_layout().fmt[(size_t) layer].tail_off == 0) {+ zc_ok.assign((size_t) n, 0);+ for (int64_t i = 0; i < n; ++i) {+ const int64_t e = experts[i];+ if (e < 0 || e >= n_expert_) continue;+ uint64_t total = 0;+ const uint64_t c = cached_bytes(layer, e, total, true);+ zc_ok[(size_t) i] = total > 0 && c >= total ? 1 : 0; // no residency query (not Linux): never+ }+ }// experts.bin with io prefetch: a blob whose pages are cached is handed out as the mapping (io_blob); only the// uncached ones are read here, together, on the fetch threadsconst bool only_uncached = io_pf_ && role_ptr_.empty() && direct_.empty();@@ -1478,7 +1655,6 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t ncold[(size_t) i] = cached_bytes(layer, e, total, false) < total ? 1 : 0;}}- const auto t0 = std::chrono::steady_clock::now();{std::lock_guard<std::mutex> lk(stage_mu_);for (int64_t i = 0; i < n; ++i) {@@ -1489,23 +1665,131 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t ncontinue; // in the RAM copyif (!override_.empty() && override_[index] != nullptr) continue;if (only_uncached && !cold[(size_t) i] && stage_of_.find(layer * n_expert_ + e) == stage_of_.end()) continue;+ if (!zc_ok.empty() && zc_ok[(size_t) i] && stage_of_.find(layer * n_expert_ + e) == stage_of_.end()) {+ zc_keys_.push_back(layer * n_expert_ + e); // read from the mapping by the kernels themselves+ continue;+ }size_t v = 0;bool fill = false;if (!claim_stage(layer * n_expert_ + e, v, fill) && fill) {todo.push_back({v, layer, e, stage_buf_[v].get()});+ if (pending != nullptr) pending[i] = 1;if (warm_stamp_) {const uint32_t s = warm_stamp_[index].load(std::memory_order_relaxed);if (s != 0 && (uint64_t) s + 3 >= epoch_ + 1) warm_hits_.fetch_add(1, std::memory_order_relaxed);}+ } else if (pending != nullptr && fill == false) {+ // being filled by another thread (a read ahead, the adaptive tier): not ready yet either+ const auto it = stage_of_.find(layer * n_expert_ + e);+ if (it != stage_of_.end() && stage_busy_[it->second]) pending[i] = 1;}}}- fill_many(todo);- if (io_pf_ && !todo.empty()) {- io_crit_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),- std::memory_order_relaxed);+}++// 149-cold STRATA_COLD_OVERLAP=1: the claimed blobs are read on background threads (the decode thread computes the+// ready experts meanwhile); the same reads as fill_many, on as many threads, each blob published as it lands.+int64_t FileExpertSource::prefetch_begin(int64_t layer, const int64_t* experts, int64_t n, uint8_t* pending) {+ for (int64_t i = 0; i < n; ++i) pending[i] = 0;+ if (!staged() || n <= 0 || layer < 0 || layer >= n_layers_) return 0;+ if (io_fill_ && io_pf_ && direct_.empty()) { // the page-fill mode: nothing is staged here, the kernels wait on pages+ prefetch(layer, experts, n);+ return 0;+ }+ prefetch_end();+ async_todo_.clear();+ async_fill_us_ = 0;+ static const int mode = [] { const char* v = std::getenv("STRATA_COLD_OVERLAP"); return v != nullptr ? std::atoi(v) : 0; }();+ claim_todo(layer, experts, n, async_todo_, pending, mode >= 3);+ if (mode == 4) { // zero-copy only: the blobs that must be read are read here, as `prefetch` reads them+ const auto t0 = std::chrono::steady_clock::now();+ fill_many(async_todo_);+ if (io_pf_ && !async_todo_.empty()) {+ io_crit_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),+ std::memory_order_relaxed);+ io_crit_n_.fetch_add(1, std::memory_order_relaxed);+ }+ async_todo_.clear();+ for (int64_t i = 0; i < n; ++i) pending[i] = 0; // one another thread still fills: `blob` waits for it+ return 0;+ }+ int64_t np = 0;+ for (int64_t i = 0; i < n; ++i) np += pending[i] ? 1 : 0;+ if (async_todo_.empty()) return np;+ async_t0_ = std::chrono::steady_clock::now();+ async_next_.store(0, std::memory_order_relaxed);+ async_end_ns_.store(0, std::memory_order_relaxed);+ const bool direct = !direct_.empty();+ auto work = [this, direct] {+ const std::vector<Fill>& todo = async_todo_;+ if (direct) {+ for (size_t at; (at = async_next_.fetch_add(16)) < todo.size();) {+ const size_t k = std::min<size_t>(16, todo.size() - at);+ const auto t0 = std::chrono::steady_clock::now();+ const bool ok = read_direct(todo.data() + at, k);+ const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();+ for (size_t i = at; i < at + k; ++i) {+ const Fill& f = todo[i];+ if (ok) publish_stage(f.v, f.layer, true, us / (double) k);+ else (void) fill_stage(f.v, f.layer, f.e, f.dst);+ }+ }+ } else {+ for (size_t i; (i = async_next_.fetch_add(1)) < todo.size();)+ (void) fill_stage(todo[i].v, todo[i].layer, todo[i].e, todo[i].dst);+ }+ const int64_t ns = (int64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(+ std::chrono::steady_clock::now() - async_t0_).count();+ int64_t cur = async_end_ns_.load(std::memory_order_relaxed);+ while (ns > cur && !async_end_ns_.compare_exchange_weak(cur, ns, std::memory_order_relaxed)) {}+ };+ const size_t nt = direct ? std::min<size_t>(4, (async_todo_.size() + 15) / 16)+ : std::min<size_t>(async_todo_.size(), (size_t) std::max(1, fetch_threads_));+ for (size_t t = 0; t < nt; ++t) async_th_.emplace_back(work);+ return np;+}++int64_t FileExpertSource::wait_landed(int64_t layer, const int64_t* experts, int64_t n, uint8_t* state) {+ std::unique_lock<std::mutex> lk(stage_mu_);+ int64_t got = 0;+ stage_cv_.wait(lk, [&] {+ bool left = false;+ for (int64_t i = 0; i < n; ++i) {+ if (state[i] != 1) continue;+ const auto it = stage_of_.find(layer * n_expert_ + experts[i]);+ // published (or failed: then `blob` reads it again on this thread, as before)+ if (it == stage_of_.end() || !stage_busy_[it->second]) { state[i] = 2; ++got; }+ else left = true;+ }+ return got > 0 || !left;+ });+ return got;+}++bool FileExpertSource::zc_roles(int64_t layer, int64_t expert, const uint8_t** p) {+ const int64_t key = layer * n_expert_ + expert;+ if (std::find(zc_keys_.begin(), zc_keys_.end(), key) == zc_keys_.end()) return false;+ for (int r = 0; r < 3; ++r) {+ const size_t i = (size_t) (3 * layer + r);+ p[r] = role_ptr_[i] + (size_t) ((uint64_t) expert * role_bytes_[i]);+ }+ file_read_bytes_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);+ if (complement_ready_) file_reads_.fetch_add(1, std::memory_order_relaxed);+ reads_.fetch_add(1, std::memory_order_relaxed);+ if (io_stats_) io_cached_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);+ return true;+}++void FileExpertSource::prefetch_end() {+ if (async_th_.empty()) return;+ for (std::thread& t : async_th_) t.join();+ async_th_.clear();+ async_fill_us_ = (double) async_end_ns_.load(std::memory_order_relaxed) / 1e3;+ if (io_pf_ && !async_todo_.empty()) { // the reads' wall time, as the synchronous fill counts it+ io_crit_us_.fetch_add((uint64_t) async_fill_us_, std::memory_order_relaxed);io_crit_n_.fetch_add(1, std::memory_order_relaxed);}+ async_todo_.clear();}void FileExpertSource::prefetch_pairs(const std::pair<int32_t, int32_t>* pairs, int64_t n) {@@ -2181,7 +2465,7 @@ void FileExpertSource::warm(int64_t layer, const int64_t* experts, int64_t n) {if (warm_stamp_) warm_stamp_[index].store(stamp, std::memory_order_relaxed);warm_count_.fetch_add(1, std::memory_order_relaxed);if (io_pf_ && direct_.empty()) { // read it ahead with pread on the I/O threads (instead of an madvise hint)- if (io_ahead_) io_enqueue(layer, e);+ if (io_ahead_) io_enqueue(layer, e, true);continue;}if (role_ptr_.empty() || unmapped_) continue;@@ -2360,6 +2644,7 @@ void RouterLookahead::run() {std::vector<float> logits((size_t) (8 * n_expert_));std::vector<int32_t> order((size_t) n_expert_);std::vector<int64_t> want;+ std::vector<float> wconf; // 149-prefetch: per wanted expert, its best rank's confidence (warm_scored)for (;;) {int64_t layer, nt;const int32_t* host_res;@@ -2382,6 +2667,8 @@ void RouterLookahead::run() {for (int dj = 0; dj < depth_ && layer0 + dj < (int64_t) routers_.size(); ++dj) {layer = layer0 + dj;want.clear();+ wconf.clear();+ const bool scored = src_->wants_scores();// The router dot is AVX2 (kq_avx2.cpp); a CPU without AVX2 (the experimental older-CPU builds) takes the AVX1// one (kq_avx1.cpp) or, without AVX, the same sums in plain C++. From the Strata_Dirigo fork (rwkeyes): an// AVX-only Xeon E5-2687W died here (vpmovzxwd) on its first request. An estimate only (which experts to@@ -2423,7 +2710,14 @@ void RouterLookahead::run() {for (int j = 0; j < kmax; ++j) {const int64_t e = order[(size_t) j];if (host_res != nullptr && host_res[(size_t) (layer * n_expert_ + e)] >= 0) continue; // on the GPU- if (j < k_ && std::find(want.begin(), want.end(), e) == want.end()) want.push_back(e);+ if (j < k_) {+ const auto it = std::find(want.begin(), want.end(), e);+ // confidence: 1 for a token's top pick down to 1/k_ for its k_-th, the best over the window's tokens,+ // halved for every layer further ahead than the next (a deeper guess is less often right)+ const float c = (float) (k_ - j) / (float) k_ * (dj == 0 ? 1.0f : std::pow(0.5f, (float) dj));+ if (it == want.end()) { want.push_back(e); wconf.push_back(c); }+ else if (scored && c > wconf[(size_t) (it - want.begin())]) wconf[(size_t) (it - want.begin())] = c;+ }if (stats_on_ && dj == 0)for (int s = 0; s < 3; ++s)if (j < ks[s]) sm[(size_t) (s * words_ + (e >> 6))] |= 1ull << (e & 63);@@ -2453,7 +2747,8 @@ void RouterLookahead::run() {pmask_[(size_t) (layer * 3 * words_ + i)].store(sm[(size_t) i], std::memory_order_relaxed);pstamp_[(size_t) layer].store(sgen + 1, std::memory_order_release);}- src_->warm(layer, want.data(), (int64_t) want.size());+ if (scored) src_->warm_scored(layer, want.data(), wconf.data(), (int64_t) want.size(), dj + 1);+ else src_->warm(layer, want.data(), (int64_t) want.size());if (fs != nullptr && dj == 0) // the next layer only: a deeper guess is too often wrong to spend a copy onfor (const int64_t e : want) fs->predict(src_, layer, (int32_t) e);predicted_.fetch_add((int64_t) want.size(), std::memory_order_relaxed);@@ -2471,11 +2766,17 @@ void FileExpertSource::begin_layer(int64_t layer, const int32_t* ids, int64_t k)(void) ids;(void) k;if (!staged()) return;- std::lock_guard<std::mutex> lk(stage_mu_);- if (layer != last_layer_) {- ++epoch_;- last_layer_ = layer;+ bool new_window = false;+ {+ std::lock_guard<std::mutex> lk(stage_mu_);+ if (layer != last_layer_) {+ ++epoch_;+ new_window = layer < last_layer_;+ last_layer_ = layer;+ io_epoch_.store(epoch_, std::memory_order_relaxed); // 149-prefetch: the workers' deadline clock+ }}+ if (new_window && io_sched_) io_window_tick();}bool FileExpertSource::transient(int64_t layer, int64_t expert) const {@@ -3465,6 +3766,7 @@ bool FileExpertSource::replace_resident(int64_t layer, int64_t in, int64_t out)const uint64_t at = complement_offsets_[i_out], bytes = layer_blob_bytes_[(size_t) layer];if (at == kNoComplement || complement_offsets_[i_in] != kNoComplement || at > complement_bytes_ ||bytes > complement_bytes_ - at) return false;+ (void) at;// read beside the copy first: a failed read leaves `out` where it wasif (ram_adapt_buf_.size() < (size_t) bytes) ram_adapt_buf_.resize((size_t) bytes);const auto t0 = std::chrono::steady_clock::now();@@ -3472,10 +3774,26 @@ bool FileExpertSource::replace_resident(int64_t layer, int64_t in, int64_t out)file_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),std::memory_order_relaxed);file_read_bytes_.fetch_add(bytes, std::memory_order_relaxed);+ return install_resident(layer, in, out, ram_adapt_buf_.data());+}++// `in`'s bytes (src) into `out`'s place of the compact copy, and the two offsets swapped - the checks again, as the caller+// may have read `src` a while ago (149-prefetch: ram_install)+bool FileExpertSource::install_resident(int64_t layer, int64_t in, int64_t out, const uint8_t* src) {+ if (!complement_ready_ || complement_host_ == nullptr || exchange_storage_.active() || !staged_.empty() ||+ layer < 0 || layer >= n_layers_ || in < 0 || out < 0 || in >= n_expert_ || out >= n_expert_ || in == out || src == nullptr)+ return false;+ const size_t i_in = (size_t) layer * (size_t) n_expert_ + (size_t) in;+ const size_t i_out = (size_t) layer * (size_t) n_expert_ + (size_t) out;+ if (i_in >= complement_offsets_.size() || i_out >= complement_offsets_.size()) return false;+ if (!override_.empty() && (override_[i_in] != nullptr || override_[i_out] != nullptr)) return false;+ const uint64_t at = complement_offsets_[i_out], bytes = layer_blob_bytes_[(size_t) layer];+ if (at == kNoComplement || complement_offsets_[i_in] != kNoComplement || at > complement_bytes_ ||+ bytes > complement_bytes_ - at) return false;// #2134: a Foresight slot may be copying out of `out`'s place right now; it is dropped (never served) before the place is rewrittenForesightSwap* fs = fs_swap_.load(std::memory_order_acquire);if (fs != nullptr) fs->hold(layer, (int32_t) out);- std::memcpy((uint8_t*) complement_host_ + (size_t) at, ram_adapt_buf_.data(), (size_t) bytes);+ std::memcpy((uint8_t*) complement_host_ + (size_t) at, src, (size_t) bytes);const bool moved = detail::exchange_cache_complement(complement_offsets_, i_out, i_in);if (fs != nullptr) fs->unhold(layer, (int32_t) out);if (!moved) return false;@@ -3483,6 +3801,119 @@ bool FileExpertSource::replace_resident(int64_t layer, int64_t in, int64_t out)return true;}+// ---- 149-prefetch: staged promotions. The read of `in` runs on this source's own thread (one at a time: the drive's+// queue stays the decode's first); the install is the decode thread's (ram_install, at a window boundary).+bool FileExpertSource::ram_stage(int64_t layer, int64_t in, int64_t out) {+ if (!complement_ready_ || complement_host_ == nullptr || layer < 0 || layer >= n_layers_ || in < 0 || out < 0 ||+ in >= n_expert_ || out >= n_expert_ || in == out)+ return false;+ {+ std::lock_guard<std::mutex> lk(ram_mu_);+ static const size_t cap = [] {+ const char* v = std::getenv("STRATA_RAM_ADAPT_QUEUE");+ return v != nullptr ? (size_t) std::max(1, std::atoi(v)) : (size_t) 64;+ }();+ if (ram_jobs_.size() >= cap) return false;+ for (const auto& j : ram_jobs_)+ if (j->layer == layer && (j->in == in || j->out == out || j->in == out || j->out == in)) return false;+ auto j = std::make_unique<RamJob>();+ j->layer = layer; j->in = in; j->out = out; j->state = 0;+ ram_jobs_.push_back(std::move(j));+ if (!ram_thread_.joinable()) {+ ram_quit_ = false;+ ram_thread_ = std::thread([this] { ram_worker(); });+ }+ }+ ram_cv_.notify_one();+ return true;+}++bool FileExpertSource::ram_queued(int64_t layer, int64_t expert) const {+ std::lock_guard<std::mutex> lk(ram_mu_);+ for (const auto& j : ram_jobs_)+ if (j->layer == layer && (j->in == expert || j->out == expert)) return true;+ return false;+}++void FileExpertSource::ram_worker() {+ for (;;) {+ RamJob* job = nullptr;+ {+ std::unique_lock<std::mutex> lk(ram_mu_);+ ram_cv_.wait(lk, [&] {+ if (ram_quit_) return true;+ for (const auto& j : ram_jobs_)+ if (j->state == 0) return true;+ return false;+ });+ if (ram_quit_) return;+ for (const auto& j : ram_jobs_)+ if (j->state == 0) { job = j.get(); break; }+ job->state = 1;+ }+ const uint64_t bytes = layer_blob_bytes_[(size_t) job->layer];+ job->buf.resize((size_t) bytes);+ const auto t0 = std::chrono::steady_clock::now();+ const bool ok = copy_from_files(job->layer, job->in, job->buf.data());+ file_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),+ std::memory_order_relaxed);+ if (ok) {+ file_read_bytes_.fetch_add(bytes, std::memory_order_relaxed);+ ram_staged_reads_.fetch_add(1, std::memory_order_relaxed);+ }+ std::lock_guard<std::mutex> lk(ram_mu_);+ job->state = ok ? 2 : 3;+ }+}++int64_t FileExpertSource::ram_install(const int32_t* host_res, int64_t n_expert) {+ std::vector<std::unique_ptr<RamJob>> done;+ {+ std::lock_guard<std::mutex> lk(ram_mu_);+ for (auto it = ram_jobs_.begin(); it != ram_jobs_.end();) {+ if ((*it)->state >= 2) { done.push_back(std::move(*it)); it = ram_jobs_.erase(it); }+ else ++it;+ }+ }+ int64_t n = 0;+ for (const auto& j : done) {+ if (j->state != 2) { ram_install_skips_.fetch_add(1, std::memory_order_relaxed); continue; }+ if (host_res != nullptr && (host_res[(size_t) (j->layer * n_expert + j->in)] >= 0 ||+ host_res[(size_t) (j->layer * n_expert + j->out)] >= 0)) {+ ram_install_gpu_.fetch_add(1, std::memory_order_relaxed); // the GPU tier took one of them meanwhile+ continue;+ }+ if (install_resident(j->layer, j->in, j->out, j->buf.data())) ++n;+ else ram_install_skips_.fetch_add(1, std::memory_order_relaxed);+ }+ return n;+}++void FileExpertSource::ram_stop() {+ {+ std::lock_guard<std::mutex> lk(ram_mu_);+ ram_quit_ = true;+ }+ ram_cv_.notify_all();+ if (ram_thread_.joinable()) ram_thread_.join();+ std::lock_guard<std::mutex> lk(ram_mu_);+ ram_jobs_.clear();+ ram_quit_ = false;+}++int FileExpertSource::tier_of(int64_t layer, int64_t expert) const {+ if (layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return 2;+ const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;+ if (complement_ready_ && resident_blob(index) != nullptr) return 0;+#if defined(__linux__)+ uint64_t total = 0;+ const uint64_t c = cached_bytes(layer, expert, total, false);+ return total > 0 && c >= total ? 1 : 2;+#else+ return 2;+#endif+}+void FileExpertSource::commit_copies() {fs_hold_staged(true); // #2134: held until commit_flip has pointed `out` at its new placeif (exchange_storage_.active()) return; // STRATA_EXCHANGE_ROTATE: ownership moves in commit_flip, nothing to copy@@ -3814,6 +4245,12 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32if (ptrace) { std::fprintf(stderr, "pool trace: layer %lld %s %lld\n", (long long) d.layers, what, a); std::fflush(stderr); }};const auto c0 = std::chrono::steady_clock::now();+ // 149-cold STRATA_COLD_OVERLAP: 0 off (default), 1 overlap, 2 measure only, 3 overlap + zero-copy, 4 zero-copy only+ static const int cold_env = [] {+ const char* v = std::getenv("STRATA_COLD_OVERLAP");+ return v == nullptr ? 0 : std::clamp(std::atoi(v), 0, 4);+ }();+ d.cold_mode = cold_env;pt("begin");d.src->begin_layer(d.layers, ids, n_tok * k);pt("begun");@@ -4012,21 +4449,56 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32}}const auto c2 = std::chrono::steady_clock::now();+ // 149-cold STRATA_COLD_OVERLAP (d.cold_mode): which of the CPU's experts still have a blob to read. Mode 1 reads+ // them in the background and computes the ready ones first, then each read one as it lands; every expert keeps+ // its own token rows (the pool splits rows, never a row's sum), and the weighted sum over a token's experts is+ // the GPU's, after the dispatch - so the result is the same bits in any batch order. The stage buffers stay+ // claimed (busy, then young) until the dispatch has joined its reads and run every batch.+ static thread_local std::vector<int64_t> miss;+ static thread_local std::vector<uint8_t> mstate; // per miss: 0 ready, 1 being read, 2 landed, 3 computed+ miss.clear();+ int64_t npend = 0;+ double cold_r_ms = 0;if (any_cpu) { // CS-T: the experts the CPU computes, fetched together (the GGUF in place reads them on several threads)- static thread_local std::vector<int64_t> miss;- miss.clear();for (int64_t i = 0; i < n_tok * k; ++i)if (kind[i] == -1 && ids[i] >= 0 && ids[i] < d.n_expert &&std::find(miss.begin(), miss.end(), (int64_t) ids[i]) == miss.end())miss.push_back(ids[i]);- d.src->prefetch(d.layers, miss.data(), (int64_t) miss.size());+ if (d.cold_mode == 0) {+ d.src->prefetch(d.layers, miss.data(), (int64_t) miss.size());+ } else {+ mstate.assign(miss.size(), 0);+ const auto r0 = std::chrono::steady_clock::now();+ npend = d.src->prefetch_begin(d.layers, miss.data(), (int64_t) miss.size(), mstate.data());+ if (npend > 0 && d.cold_mode == 2) { // measure: every read first, as mode 0 does+ d.src->prefetch_end();+ for (uint8_t& s : mstate) if (s == 1) s = 2;+ cold_r_ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - r0).count();+ d.ms_cold_wait += cold_r_ms;+ }+ }}- int njobs = 0;+ // the CPU's experts this batch takes: all (mode 0, or no read), else by their state (want 0: the ready ones,+ // want 2: those that landed since the last batch)+ auto take = [&](int64_t e, uint8_t want) {+ if (npend == 0) return true;+ for (size_t q = 0; q < miss.size(); ++q)+ if (miss[q] == e) return mstate[q] == want;+ return want == 0;+ };+ int njobs = 0, njobs_all = 0;+ bool first_pass = true;+ uint8_t want = 0;+ double run_ms_extra = 0, cr_ms = 0;+ for (;;) {for (int64_t t = 0; t < n_tok; ++t)for (int64_t j = 0; j < k; ++j) {const int64_t i = t * k + j;const int64_t e = ids[i];float* row = out + (size_t) i * H;+ if (!first_pass && (e < 0 || e >= d.n_expert || kind[i] != -1 || !take(e, want)))+ continue; // a later batch: only the CPU entries of the experts that just landed+ if (first_pass && kind[i] == -1 && e >= 0 && e < d.n_expert && !take(e, 0)) continue; // being read: a later batchif (e < 0 || e >= d.n_expert) {d.failed = true;d.fail = "a routed expert id is out of range";@@ -4053,7 +4525,9 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32++d.cache_refused;int16_t& jo = d.job_of[(size_t) e];if (jo < 0) {- const uint8_t* b = d.src->blob(d.layers, e);+ const uint8_t* zp[3] = {nullptr, nullptr, nullptr};+ const bool zc = d.cold_mode >= 3 && native && d.src->zc_roles(d.layers, e, zp);+ const uint8_t* b = zc ? zp[0] : d.src->blob(d.layers, e);if (b == nullptr) {d.failed = true;d.fail = "the expert source could not produce a blob";@@ -4066,6 +4540,16 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32ExpertJobMulti& nj = d.jobs_multi[(size_t) jo];nj.blob = b;nj.nt = 0;+ nj.fmt = nullptr;+ if (zc) { // 149-cold mode 3: the gate rows are the blob; up and down at their distance in the mapping+ if (d.zc_fmt.size() < d.jobs_multi.size()) d.zc_fmt.resize(d.jobs_multi.size());+ strata::kernels::cpu::NativeFmt& f = d.zc_fmt[(size_t) jo];+ f = lay.fmt[(size_t) d.layers];+ f.up_off = (size_t) ((uintptr_t) zp[1] - (uintptr_t) zp[0]); // modular: the tensors' order+ f.down_off = (size_t) ((uintptr_t) zp[2] - (uintptr_t) zp[0]); // in the file does not matter+ nj.fmt = &f;+ ++d.cold_zc;+ }}ExpertJobMulti& jb = d.jobs_multi[(size_t) jo];jb.act[jb.nt] = &d.act_multi[(size_t) t];@@ -4074,10 +4558,56 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32++jb.nt;++d.multi_entries;}+ if (npend == 0) break;+ // 149-cold: run this batch now (the ready experts, then each landed group), then wait for the next landing+ {+ const auto b0 = std::chrono::steady_clock::now();+ if (njobs > 0) {+ if (native) d.pool->run_split_multi_native(lay.fmt[(size_t) d.layers], d.jobs_multi.data(), njobs);+ else d.pool->run_split_multi(d.jobs_multi.data(), njobs);+ ++d.cold_batches;+ }+ const auto b1 = std::chrono::steady_clock::now();+ const double bm = std::chrono::duration<double, std::milli>(b1 - b0).count();+ run_ms_extra += bm;+ if (first_pass) { d.ms_cold_cr += bm; cr_ms = bm; }+ else d.ms_cold_cc += bm;+ if (first_pass) d.cold_ready += njobs;+ // the computed ones' job slots are free again (an expert is in one batch only)+ for (int jx = 0; jx < njobs; ++jx) d.jobs_multi[(size_t) jx].nt = 0;+ for (int64_t i = 0; i < n_tok * k; ++i)+ if (ids[i] >= 0 && ids[i] < d.n_expert) d.job_of[(size_t) ids[i]] = -1;+ if (!first_pass) for (uint8_t& s : mstate) if (s == 2) s = 3;+ njobs_all += njobs;+ njobs = 0;+ first_pass = false;+ want = 2;+ bool left = false;+ for (uint8_t s : mstate) left = left || s == 1 || s == 2;+ if (!left) break;+ if (std::none_of(mstate.begin(), mstate.end(), [](uint8_t s) { return s == 2; })) {+ const auto w0 = std::chrono::steady_clock::now();+ (void) d.src->wait_landed(d.layers, miss.data(), (int64_t) miss.size(), mstate.data());+ d.ms_cold_wait += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - w0).count();+ }+ }+ }+ if (npend > 0) {+ d.src->prefetch_end();+ const FileExpertSource* fs = dynamic_cast<const FileExpertSource*>(d.src);+ const double r_ms = d.cold_mode == 2 ? cold_r_ms : (fs != nullptr ? fs->async_fill_us() / 1e3 : 0.0);+ ++d.cold_layers;+ d.cold_pending += npend;+ d.ms_cold_r += r_ms;+ d.ms_cold_hide += std::min(r_ms, cr_ms); // what an overlap can hide of this layer's reads at most+ njobs = njobs_all;+ d.ms_run += run_ms_extra; // the batches ran inside the job loop: the pool's time stays the pool's+ d.ms_jobs -= run_ms_extra;+ }const auto c3 = std::chrono::steady_clock::now();if (d.plan != nullptr) d.plan->cpu_jobs = njobs;pt("run", njobs);- if (njobs > 0) {+ if (njobs > 0 && npend == 0) {if (native) d.pool->run_split_multi_native(lay.fmt[(size_t) d.layers], d.jobs_multi.data(), njobs);else d.pool->run_split_multi(d.jobs_multi.data(), njobs);}diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cppindex 9641ba6a9..d9961938a 100644--- a/src/kernels/cpu/pool.cpp+++ b/src/kernels/cpu/pool.cpp@@ -714,32 +714,33 @@ void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {const int e = (int) (r / per), r0 = (int) (r % per);const int r1 = (int) std::min<int64_t>(per, r0 + (g1 - r));SplitBufMulti& sb = split_multi_[(size_t) e];- if (mode_ == 5 && q2_native_kernels(nfmt_->gu_type)) {+ const NativeFmt* const nfmt = mjobs_[e].fmt != nullptr ? mjobs_[e].fmt : nfmt_; // 149-cold: same types and rows+ if (mode_ == 5 && q2_native_kernels(nfmt->gu_type)) {// a native Q2_0 pack: gate and up rows on the Q2_0 kernels, then SwiGLUthread_local float gbuf[MAXT][FF], ubuf[MAXT][FF];float* gp[MAXT];float* up[MAXT];for (int t = 0; t < mjobs_[e].nt; ++t) { gp[t] = gbuf[t]; up[t] = ubuf[t]; }- const int nbk = (int) (nfmt_->n_embd / 64);- q2_rows_any(mjobs_[e].blob, nfmt_->gu_row, nbk, mjobs_[e].act, mjobs_[e].nt, gp, r0, r1);- q2_rows_any(mjobs_[e].blob + nfmt_->up_off, nfmt_->gu_row, nbk, mjobs_[e].act, mjobs_[e].nt, up, r0, r1);+ const int nbk = (int) (nfmt->n_embd / 64);+ q2_rows_any(mjobs_[e].blob, nfmt->gu_row, nbk, mjobs_[e].act, mjobs_[e].nt, gp, r0, r1);+ q2_rows_any(mjobs_[e].blob + nfmt->up_off, nfmt->gu_row, nbk, mjobs_[e].act, mjobs_[e].nt, up, r0, r1);for (int t = 0; t < mjobs_[e].nt; ++t)for (int r = r0; r < r1; ++r)sb.ff[t][r] = (gbuf[t][r] / (1.f + std::exp(-gbuf[t][r]))) * ubuf[t][r];} else if (mode_ == 5) {float* ff[MAXT];for (int t = 0; t < mjobs_[e].nt; ++t) ff[t] = sb.ff[t];- native_gu_rows(*nfmt_, mjobs_[e].blob, mjobs_[e].nact, mjobs_[e].nt, ff, r0, r1);- } else if (q2_native_kernels(nfmt_->d_type)) {+ native_gu_rows(*nfmt, mjobs_[e].blob, mjobs_[e].nact, mjobs_[e].nt, ff, r0, r1);+ } else if (q2_native_kernels(nfmt->d_type)) {// Q2_0 down (most IQ layers): the AVX-512 kernel, ggml-cpu has only a scalar one on x86const ActQ* a2[MAXT];for (int t = 0; t < mjobs_[e].nt; ++t) a2[t] = &sb.a2[t];- q2_rows_any(mjobs_[e].blob + nfmt_->down_off, nfmt_->d_row, (int) (nfmt_->n_ff / 64), a2,+ q2_rows_any(mjobs_[e].blob + nfmt->down_off, nfmt->d_row, (int) (nfmt->n_ff / 64), a2,mjobs_[e].nt, mjobs_[e].out, r0, r1);} else {const void* hq[MAXT];for (int t = 0; t < mjobs_[e].nt; ++t) hq[t] = sb.hq[t];- native_down_rows(*nfmt_, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);+ native_down_rows(*nfmt, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);}r += r1 - r0;}diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cppindex bceacbda3..4c8736f20 100644--- a/src/prefill/prefill.cpp+++ b/src/prefill/prefill.cpp@@ -3568,6 +3568,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stril, p0);cudaStreamSynchronize(m.cs);core::progress_at("reading the prompt (batched): layer", l, p0);+ if (on_routes) on_routes(l, ids_h, T * K); // 149-prefetch: the RAM policy's prompt seedif (cpu_maybe) { // the rows buffer, once, at its largest (an expert takes at most MAXT tokens' rows)const size_t cap = (size_t) std::min<int64_t>(T * K, (int64_t) strata::kernels::cpu::MAXT * m.g->n_expert) * N;if (m.cpu_rows_n < cap) {diff --git a/src/program/generate.cpp b/src/program/generate.cppindex aae516316..06a2281bd 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -5454,7 +5454,10 @@ int main(int argc, char** argv) {std::fprintf(stderr, "strata generate: file tier io prefetch ON (STRATA_IO_PREFETCH=1): whole-blob preads, ""the layer's uncached experts and the predicted ones read by %s I/O threads (STRATA_IO_PF_THREADS, default 8)\n",th != nullptr ? th : "8");- else+ if (src.io_prefetch() && src.io_sched())+ std::fprintf(stderr, "strata generate: io prefetch scheduling ON (STRATA_IO_PF_SCHED=1): a deadline per read, "+ "confidence admission (STRATA_IO_PF_MINCONF), a per-window byte budget (STRATA_IO_PF_BUDGET_MB / _FRAC)\n");+ if (!src.io_prefetch())std::fprintf(stderr, "strata generate: STRATA_IO_PREFETCH=1 is not available here (Linux mapped file tier ""only, not with unbuffered reads); the file tier reads as before\n");if (src.io_prefetch()) { // recommend, never force: the engine runs as asked@@ -7868,6 +7871,23 @@ int main(int argc, char** argv) {"cache (STRATA_PREFILL_HELPER_SOURCE)\n");}strata::prefill::Prefill sp;+ // 149-prefetch STRATA_RAM_ADAPT2=1 (with STRATA_RAM_ADAPT=n, the promotions staged per window): see the window loop+ struct RamAdapt2 {+ bool on = false;+ float prompt_w = 0.25f, cached_w = 0.15f;+ int per_layer = 2;+ int skip_top = 2;+ int64_t installed = 0;+ std::vector<float> seed;+ } ram_adapt2;+ {+ const char* v = std::getenv("STRATA_RAM_ADAPT2");+ ram_adapt2.on = v != nullptr && std::atoi(v) != 0;+ if (const char* w = std::getenv("STRATA_RAM_ADAPT_PROMPT_W")) ram_adapt2.prompt_w = (float) std::max(0.0, std::atof(w));+ if (const char* w = std::getenv("STRATA_RAM_ADAPT_CACHED_W")) ram_adapt2.cached_w = (float) std::clamp(std::atof(w), 0.0, 1.0);+ if (const char* w = std::getenv("STRATA_RAM_ADAPT_PER_LAYER")) ram_adapt2.per_layer = std::max(1, std::atoi(w));+ if (const char* w = std::getenv("STRATA_RAM_ADAPT_SKIP_TOP")) ram_adapt2.skip_top = std::max(0, std::atoi(w));+ }// the pool is idle while a prompt is read unless batch slots decode between its parts; with// STRATA_PREFILL_CPU_SHARE the staged-chunk limit before the chunk below sizes the loans (bytes_needed reads it)// #1607: Windows, the commit limit (RAM + page file) is what ends the engine silently; compare what this run plans@@ -10367,6 +10387,19 @@ int main(int argc, char** argv) {return true;};sp.should_stop = [&] { return stop_req.load(); };+ // 149-prefetch STRATA_RAM_ADAPT2: the prompt's routing seeds the RAM policy with a limited weight+ // (STRATA_RAM_ADAPT_PROMPT_W, default 0.25: an expert every prompt token routes to counts like ~5 decode routings),+ // fading by 5% a decode window; the decode's own counts and the GPU tier are untouched+ if (ram_adapt2.on && ram_adapt2.prompt_w > 0.0f) {+ ram_adapt2.seed.assign((size_t) (g.n_layers * g.n_expert), 0.0f);+ sp.on_routes = [&](int64_t l, const int32_t* ids, int64_t n) {+ if (l < 0 || l >= g.n_layers || n <= 0) return;+ const float per = ram_adapt2.prompt_w * 20.0f * (float) g.n_expert_used / (float) n; // = w x 20 / tokens+ float* sd = ram_adapt2.seed.data() + l * g.n_expert;+ for (int64_t i = 0; i < n; ++i)+ if (ids[i] >= 0 && ids[i] < g.n_expert) sd[ids[i]] += per;+ };+ }// STRATA_TRACE=1: one stderr line per step of a request (the log shows where a request stops)const bool trace = std::getenv("STRATA_TRACE") != nullptr;auto tr = [&](const char* what, long long a = -1, long long b = -1) {@@ -14276,6 +14309,79 @@ int main(int argc, char** argv) {const char* e = std::getenv("STRATA_RAM_ADAPT");return e != nullptr ? std::max(0, std::atoi(e)) : 0;}();+ if (ram_adapt_n > 0 && ram_adapt2.on && !first_window && !drive.d.usage.empty() && pending.empty() &&+ (!ajob || astate == AState::Idle) && src.complement_ready() && !multi_gpu && !peer.valid() &&+ !remote_opt && o.batch == 0) {+ // 149-prefetch STRATA_RAM_ADAPT2=1: install the promotions whose reads have landed (window boundary:+ // nothing reads the RAM copy now), then stage the next ones, ranked by the stall they are expected to+ // save per byte: (decayed decode count + the prompt's seed) x the miss's cost (a cold expert costs+ // a drive read, one in the page cache STRATA_RAM_ADAPT_CACHED_W of that), against what the evicted+ // one is expected to cost; up to STRATA_RAM_ADAPT_PER_LAYER pairs a layer. The reads run on the+ // source's I/O thread, not here.+ ram_adapt2.installed += src.ram_install(host_res.data(), g.n_expert);+ for (float& x : ram_adapt2.seed) x *= 0.95f;+ struct Cand { float score; int32_t e; };+ struct RamSwap2 { float gain; int32_t layer, in, out; };+ std::vector<RamSwap2> swaps;+ std::vector<Cand> ins, outs;+ for (int64_t l = 0; l < g.n_layers; ++l) {+ const float* u = drive.d.usage.data() + l * g.n_expert;+ const float* sd = ram_adapt2.seed.empty() ? nullptr : ram_adapt2.seed.data() + l * g.n_expert;+ const int32_t* r = host_res.data() + l * g.n_expert;+ ins.clear();+ outs.clear();+ for (int32_t e = 0; e < (int32_t) g.n_expert; ++e) {+ if (r[e] >= 0) continue; // in VRAM+ const float ue = u[e] + (sd != nullptr ? sd[e] : 0.0f);+ if (src.has_resident(l, e)) outs.push_back({ue, e});+ else if (ue >= 2.0f) ins.push_back({ue, e});+ }+ // the GPU tier's own next picks are left to it: its round takes the layer's most used experts+ // outside VRAM (about 2 a layer a round), and a RAM place spent on one of them is skipped at the+ // install (measured: 60% of the promotions skipped before this rule) - STRATA_RAM_ADAPT_SKIP_TOP+ if (ram_adapt2.skip_top > 0 && !ins.empty()) {+ std::vector<float> du;+ for (const Cand& c : ins) du.push_back(u[c.e]);+ const size_t k = std::min(du.size(), (size_t) ram_adapt2.skip_top);+ std::nth_element(du.begin(), du.begin() + (std::ptrdiff_t) (k - 1), du.end(), std::greater<float>());+ const float top = du[k - 1];+ size_t dropped = 0;+ ins.erase(std::remove_if(ins.begin(), ins.end(), [&](const Cand& c) {+ if (dropped < k && u[c.e] >= top) { ++dropped; return true; }+ return false;+ }), ins.end());+ }+ if (ins.empty() || outs.empty()) continue;+ const size_t P = (size_t) ram_adapt2.per_layer;+ const size_t ni = std::min(ins.size(), 2 * P), no = std::min(outs.size(), P);+ std::partial_sort(ins.begin(), ins.begin() + (std::ptrdiff_t) ni, ins.end(),+ [](const Cand& a, const Cand& b) { return a.score > b.score; });+ std::partial_sort(outs.begin(), outs.begin() + (std::ptrdiff_t) no, outs.end(),+ [](const Cand& a, const Cand& b) { return a.score < b.score; });+ ins.resize(ni);+ for (Cand& c : ins) { // the miss cost of the few best: cold on the drive, or in the page cache+ if (src.ram_queued(l, c.e)) { c.score = -1.0f; continue; }+ c.score *= src.tier_of(l, c.e) == 1 ? ram_adapt2.cached_w : 1.0f;+ }+ std::sort(ins.begin(), ins.end(), [](const Cand& a, const Cand& b) { return a.score > b.score; });+ const double bytes = (double) std::max<uint64_t>(1, src.blob_bytes(l));+ size_t oi = 0;+ for (size_t k = 0; k < ins.size() && oi < no; ++k) {+ while (oi < no && src.ram_queued(l, outs[oi].e)) ++oi;+ if (oi >= no) break;+ const float gain = ins[k].score - outs[oi].score; // the evicted one would cost a drive read+ if (ins[k].score <= 0.0f || gain < 1.0f) break;+ swaps.push_back({(float) (gain / bytes * 1048576.0), (int32_t) l, ins[k].e, outs[oi].e});+ ++oi;+ }+ }+ std::sort(swaps.begin(), swaps.end(), [](const RamSwap2& a, const RamSwap2& b) { return a.gain > b.gain; });+ int staged_n = 0;+ for (const RamSwap2& x : swaps) {+ if (staged_n >= ram_adapt_n) break;+ if (src.ram_stage(x.layer, x.in, x.out)) ++staged_n;+ }+ } elseif (ram_adapt_n > 0 && !first_window && !drive.d.usage.empty() && pending.empty() &&(!ajob || astate == AState::Idle) && src.complement_ready() && !multi_gpu && !peer.valid() &&!remote_opt && o.batch == 0) {@@ -14763,6 +14869,11 @@ int main(int argc, char** argv) {"VRAM tier, %lld blob reads from the file\n",(double) src.resident_bytes() / 1073741824.0, (long long) src.exchanges(),(long long) src.file_reads());+ if (ram_adapt2.on) // 149-prefetch: the staged promotions, cumulative+ std::fprintf(stderr, "strata serve: resident RAM (STRATA_RAM_ADAPT2): %lld promotions read on the I/O thread, "+ "%lld installed, %lld skipped at the install (%lld of them: the GPU tier took one of the pair)\n",+ (long long) src.ram_staged_reads(),+ (long long) ram_adapt2.installed, (long long) (src.ram_install_skips() + src.ram_install_gpu()), (long long) src.ram_install_gpu());if (src.ram_adapts() > 0) // STRATA_RAM_ADAPT, cumulativestd::fprintf(stderr, "strata serve: resident RAM: %lld experts moved in from the file in place of ""unused ones (STRATA_RAM_ADAPT)\n", (long long) src.ram_adapts());@@ -14814,12 +14925,39 @@ int main(int argc, char** argv) {(unsigned long long) (ic.pf_dropped - ic_prev.pf_dropped),(double) (ic.crit_us - ic_prev.crit_us) / 1e3,(unsigned long long) (ic.crit_n - ic_prev.crit_n));+ if (src.io_prefetch()) // 149-prefetch: bytes read ahead, of them used before their demand; the scheduler's drops+ std::fprintf(stderr, "; read ahead %.1f MB, %.1f MB of it used; %llu late, %llu not admitted (confidence), "+ "%llu (budget %.0f MB/window)",+ (double) (ic.pf_bytes - ic_prev.pf_bytes) / 1e6,+ (double) (ic.pf_used_bytes - ic_prev.pf_used_bytes) / 1e6,+ (unsigned long long) (ic.pf_late - ic_prev.pf_late),+ (unsigned long long) (ic.pf_rej_conf - ic_prev.pf_rej_conf),+ (unsigned long long) (ic.pf_rej_budget - ic_prev.pf_rej_budget),+ (double) ic.pf_budget_bytes / 1048576.0);std::fprintf(stderr, "\n");io_prev = io_now;ic_prev = ic;fb_prev = fb;}}+ if (drive.d.cold_mode != 0) { // 149-cold STRATA_COLD_OVERLAP: this request's layers that had a blob to read+ static strata::core::ExpertDispatch cp;+ const strata::core::ExpertDispatch& c = drive.d;+ const double nl = (double) std::max<int64_t>(1, c.cold_layers - cp.cold_layers);+ std::fprintf(stderr, "strata serve: cold reads (STRATA_COLD_OVERLAP=%d) this request: %lld experts from the mapping "+ "directly; %lld layers with a blob to "+ "read (%lld such experts, %lld ready beside them, %lld pool batches); per such layer: "+ "R %.3f ms, C_r %.3f ms, C_c %.3f ms, the decode waited %.3f ms; total waited %.1f ms, min(R, C_r) %.1f ms\n",+ c.cold_mode, (long long) (c.cold_zc - cp.cold_zc), (long long) (c.cold_layers - cp.cold_layers),+ (long long) (c.cold_pending - cp.cold_pending), (long long) (c.cold_ready - cp.cold_ready),+ (long long) (c.cold_batches - cp.cold_batches),+ (c.ms_cold_r - cp.ms_cold_r) / nl, (c.ms_cold_cr - cp.ms_cold_cr) / nl,+ (c.ms_cold_cc - cp.ms_cold_cc) / nl, (c.ms_cold_wait - cp.ms_cold_wait) / nl,+ c.ms_cold_wait - cp.ms_cold_wait, c.ms_cold_hide - cp.ms_cold_hide);+ cp.cold_layers = c.cold_layers; cp.cold_pending = c.cold_pending; cp.cold_ready = c.cold_ready; cp.cold_zc = c.cold_zc;+ cp.cold_batches = c.cold_batches; cp.ms_cold_r = c.ms_cold_r; cp.ms_cold_cr = c.ms_cold_cr;+ cp.ms_cold_cc = c.ms_cold_cc; cp.ms_cold_wait = c.ms_cold_wait; cp.ms_cold_hide = c.ms_cold_hide;+ }// STRATA_SPLIT_TIMING: where each verify stage's host time went, cumulative per window since the start// (waiting for its GPU to ring a layer, the CPU pool and plan per layer, staging the window)if (static const bool st_timing = std::getenv("STRATA_SPLIT_TIMING") != nullptr; st_timing)