Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

149: io prefetch scheduling, RAM_ADAPT2, cold-read overlap and zero-copy GGUF experts (all opt-in)

Niko1221c85cc6dd19d2
diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hpp
index 4850b8b6a..388358d5f 100644
--- a/include/strata/core/expert_source.hpp
+++ b/include/strata/core/expert_source.hpp
@@ -30,6 +30,7 @@
#include <algorithm>
#include <cmath>
#include <atomic>
+#include <chrono>
#include <cstdint>
#include <cstddef>
#include <condition_variable>
@@ -233,9 +234,37 @@ public:
/// The `n` experts of `layer` the CPU is about to ask `blob` for, all at once: a source that reads a file may
/// fetch them in parallel. The bytes `blob` then returns are the same. Default: nothing.
virtual void prefetch(int64_t layer, const int64_t* experts, int64_t n) { (void) layer; (void) experts; (void) n; }
+ /// 149-cold (STRATA_COLD_OVERLAP): `prefetch`, but the blobs that must be read are read in the background.
+ /// `pending[i]` is set to 1 for each expert whose blob is not ready yet (0: `blob` returns it at once); returns how
+ /// many. `wait_landed` then waits until at least one pending expert's blob is ready (its state goes 1 -> 2) and
+ /// returns how many landed; `prefetch_end` joins the reads (every claimed blob is published by then). `blob` on
+ /// a pending expert still works: it waits for that blob. Default: the synchronous `prefetch`, nothing pending.
+ virtual int64_t prefetch_begin(int64_t layer, const int64_t* experts, int64_t n, uint8_t* pending) {
+ prefetch(layer, experts, n);
+ for (int64_t i = 0; i < n; ++i) pending[i] = 0;
+ return 0;
+ }
+ virtual int64_t wait_landed(int64_t layer, const int64_t* experts, int64_t n, uint8_t* state) {
+ (void) layer; (void) experts;
+ int64_t got = 0;
+ for (int64_t i = 0; i < n; ++i)
+ if (state[i] == 1) { state[i] = 2; ++got; }
+ return got;
+ }
+ virtual void prefetch_end() {}
+ /// 149-cold STRATA_COLD_OVERLAP=3: the last prefetch_begin left this expert unstaged because the page cache holds
+ /// all of it (the GGUF read in place): its gate, up and down rows in the mapping. False: ask `blob`.
+ virtual bool zc_roles(int64_t layer, int64_t expert, const uint8_t** p) { (void) layer; (void) expert; (void) p; return false; }
/// CS-T: the experts of `layer` a predictor expects next - a source that reads a file may start reading their
/// pages now, in the background. Only warms: what `blob` returns is unchanged. Default: nothing.
virtual void warm(int64_t layer, const int64_t* experts, int64_t n) { (void) layer; (void) experts; (void) n; }
+ /// 149-prefetch (STRATA_IO_PF_SCHED=1): `warm` with each expert's confidence (0..1) and how many layers ahead of the
+ /// decode its layer is (the deadline: past it the read is dropped). Default: plain `warm`.
+ virtual void warm_scored(int64_t layer, const int64_t* experts, const float* conf, int64_t n, int ahead) {
+ (void) conf; (void) ahead; warm(layer, experts, n);
+ }
+ /// Whether the predictor should compute the confidences for warm_scored (else it calls plain warm).
+ virtual bool wants_scores() const { return false; }
/// Whether `warm` does anything (the predictor is not run otherwise).
virtual bool warms() const { return false; }
protected:
@@ -564,6 +593,19 @@ struct ExpertDispatch {
int64_t tail_skipped = 0;
int64_t tail_experts = 0;
double ms_plan = 0, ms_actq = 0, ms_jobs = 0, ms_run = 0; ///< verify-window dispatch sections
+ /// 149-cold STRATA_COLD_OVERLAP: 0 off (the CPU starts once every blob of the layer is read), 1 the blobs that
+ /// must be read are read in the background while the pool computes the ready experts, each landed one computed
+ /// as it lands; 2 measure only (read everything first, then the ready experts, then the read ones, timed apart).
+ /// The per-layer counters: layers with a blob to read, those experts, the ready ones beside them; R = the reads'
+ /// wall time, C_r / C_c = the pool on the ready / the read experts, wait = what the decode thread waited for
+ /// reads, overlap_max = the sum over layers of min(R, C_r) (mode 2: the stall an overlap could hide).
+ /// 3 = 1 + zero-copy: an expert the page cache holds whole (the GGUF in place, Linux mincore) is computed from the
+ /// mapping instead of a stage buffer; 4 = zero-copy with the reads as in mode 0. Same bytes, same bits.
+ int cold_mode = 0;
+ std::vector<strata::kernels::cpu::NativeFmt> zc_fmt; ///< mode 3: the per-job layouts of the zero-copy experts
+ int64_t cold_zc = 0; ///< mode 3: experts computed from the mapping directly
+ int64_t cold_layers = 0, cold_pending = 0, cold_ready = 0, cold_batches = 0;
+ double ms_cold_r = 0, ms_cold_cr = 0, ms_cold_cc = 0, ms_cold_wait = 0, ms_cold_hide = 0;
/// Plan v0.3 P6: decayed routing counts per (layer, expert) during decode (sized by the caller; empty = off),
/// which the driver uses to swap the most-routed missing experts into the VRAM tier between rounds.
std::vector<float> usage;
@@ -756,6 +798,22 @@ public:
/// STRATA_EXCHANGE_ROTATE (false, nothing changes).
bool replace_resident(int64_t layer, int64_t in, int64_t out);
int64_t ram_adapts() const { return ram_adapts_; }
+ // ---- 149-prefetch (STRATA_RAM_ADAPT2): the same promotion, staged. `ram_stage` queues the read of `in` on an I/O thread
+ // (false: already queued, the queue is full, or no compact copy); `ram_install` - called by the decode thread at a window
+ // boundary, when no window reads the RAM copy - puts every read that has landed into `out`'s place, unless the GPU tier
+ // took `in` or `out` meanwhile (`host_res`: the residency table, n_expert per layer) or the copy no longer fits.
+ bool ram_stage(int64_t layer, int64_t in, int64_t out);
+ int64_t ram_install(const int32_t* host_res, int64_t n_expert);
+ bool ram_queued(int64_t layer, int64_t expert) const; ///< `expert` is the `in` or `out` of a staged promotion
+ int64_t ram_staged_reads() const { return ram_staged_reads_.load(std::memory_order_relaxed); }
+ int64_t ram_install_skips() const { return ram_install_skips_.load(std::memory_order_relaxed); }
+ int64_t ram_install_gpu() const { return ram_install_gpu_.load(std::memory_order_relaxed); } ///< skipped: the GPU tier took in or out
+ /// Where an expert outside the GPU is now: 0 the RAM copy, 1 the files with its pages in the page cache, 2 the files, cold
+ /// (Linux: four sampled pages; elsewhere 2).
+ int tier_of(int64_t layer, int64_t expert) const;
+ uint64_t blob_bytes(int64_t layer) const {
+ return layer >= 0 && (size_t) layer < layer_blob_bytes_.size() ? layer_blob_bytes_[(size_t) layer] : 0;
+ }
bool exchange_rotation() const { return exchange_storage_.active(); }
uint64_t rotated_exchanges() const { return exchange_storage_.exchanges(); }
uint64_t avoided_exchange_copy_bytes() const { return exchange_storage_.avoided_bytes(); }
@@ -832,9 +890,20 @@ public:
void begin_layer(int64_t layer, const int32_t* ids, int64_t k) override;
/// CS-T: the GGUF in place assembles the missed experts on `fetch_threads_` threads.
void prefetch(int64_t layer, const int64_t* experts, int64_t n) override;
+ int64_t prefetch_begin(int64_t layer, const int64_t* experts, int64_t n, uint8_t* pending) override;
+ int64_t wait_landed(int64_t layer, const int64_t* experts, int64_t n, uint8_t* state) override;
+ void prefetch_end() override;
+ bool zc_roles(int64_t layer, int64_t expert, const uint8_t** p) override;
+ /// 149-cold: the wall time of the last background read batch (prefetch_begin -> its last blob published), us
+ double async_fill_us() const { return async_fill_us_; }
/// CS-T: the GGUF in place asks the OS for the predicted experts' pages (PrefetchVirtualMemory on Windows,
/// madvise(WILLNEED) elsewhere), skipping the RAM copy's.
void warm(int64_t layer, const int64_t* experts, int64_t n) override;
+ /// 149-prefetch: the scheduled read-ahead (STRATA_IO_PF_SCHED=1 with STRATA_IO_PREFETCH=1): admission by confidence
+ /// against the queue's pressure, a speculative byte budget per decode window, a deadline per read.
+ void warm_scored(int64_t layer, const int64_t* experts, const float* conf, int64_t n, int ahead) override;
+ bool io_sched() const { return io_sched_; }
+ bool wants_scores() const override { return io_sched_ && io_pf_; }
/// Not when the reads are unbuffered: the warmed pages would be read through the file cache, a second time.
bool warms() const override { return (!role_ptr_.empty() || io_pf_) && direct_.empty(); }
/// Of the blobs the file tier read for the decode, how many had been warmed for their layer beforehand.
@@ -874,6 +943,12 @@ public:
uint64_t pf_used = 0, pf_unused = 0; ///< prefetched blobs a layer then used / never used
uint64_t crit_us = 0; ///< wall time the decode waited on file reads (prefetch + staged_blob)
uint64_t crit_n = 0;
+ // 149-prefetch: the read-ahead's bytes and what came of them
+ uint64_t pf_bytes = 0; ///< bytes the read-ahead workers read (all modes)
+ uint64_t pf_used_bytes = 0; ///< of them, blobs a layer then asked for (arrived before demand)
+ uint64_t pf_late = 0; ///< reads dropped because the decode had reached their layer
+ uint64_t pf_rej_conf = 0, pf_rej_budget = 0; ///< predictions not admitted (STRATA_IO_PF_SCHED)
+ uint64_t pf_budget_bytes = 0; ///< the current per-window speculative budget (scheduler)
};
IoCounters io_counters() const;
@@ -890,6 +965,16 @@ private:
struct Fill { size_t v; int64_t layer, e; uint8_t* dst; };
/// The claimed buffers' blobs: one overlapped batch when unbuffered, else the fetch threads' mapped copies.
void fill_many(const std::vector<Fill>& todo);
+ /// 149-cold: the claim half of `prefetch` (the blobs this call must fill; pending[i] = 1 for those, if given)
+ void claim_todo(int64_t layer, const int64_t* experts, int64_t n, std::vector<Fill>& todo, uint8_t* pending,
+ bool zc = false);
+ std::vector<int64_t> zc_keys_; ///< 149-cold mode 3: the keys the last prefetch_begin left in the mapping
+ std::vector<Fill> async_todo_;
+ std::vector<std::thread> async_th_;
+ std::atomic<size_t> async_next_{0};
+ std::atomic<int64_t> async_end_ns_{0};
+ std::chrono::steady_clock::time_point async_t0_{};
+ double async_fill_us_ = 0;
/// #286: the blobs from the drive, unbuffered: every role's 4 KiB-aligned window is read at once (overlapped on
/// Windows, io_submit on Linux) into this thread's aligned buffer, then copied into place. False when a read fails.
bool read_direct(const Fill* fills, size_t n) const;
@@ -947,6 +1032,17 @@ private:
uint64_t stage_seq_ = 0;
int64_t ram_adapts_ = 0; ///< replace_resident: experts moved from the file into the compact copy
std::vector<uint8_t> ram_adapt_buf_; ///< ...one blob, read here before it replaces another
+ // 149-prefetch: the staged promotions (ram_stage / ram_install)
+ struct RamJob { int64_t layer, in, out; std::vector<uint8_t> buf; int state; }; ///< state 0 queued, 1 reading, 2 landed, 3 failed
+ mutable std::mutex ram_mu_;
+ std::condition_variable ram_cv_;
+ std::deque<std::unique_ptr<RamJob>> ram_jobs_;
+ std::thread ram_thread_;
+ bool ram_quit_ = false;
+ std::atomic<int64_t> ram_staged_reads_{0}, ram_install_skips_{0}, ram_install_gpu_{0};
+ bool install_resident(int64_t layer, int64_t in, int64_t out, const uint8_t* src); ///< the second half of replace_resident
+ void ram_worker();
+ void ram_stop();
uint64_t epoch_ = 0;
int64_t last_layer_ = -1;
bool stage_grew_ = false;
@@ -971,6 +1067,27 @@ private:
std::condition_variable io_cv_;
std::deque<std::pair<int32_t, int32_t>> io_q_; ///< (layer, expert) to read ahead
bool io_quit_ = false;
+ // ---- 149-prefetch: the scheduled read-ahead (STRATA_IO_PF_SCHED=1)
+ struct IoJob { int32_t layer, expert; float conf; uint64_t deadline; uint64_t bytes; };
+ bool io_sched_ = false;
+ float io_minconf_ = 0.5f; ///< STRATA_IO_PF_MINCONF: the bar with an empty queue
+ uint64_t io_budget_fixed_ = 0; ///< STRATA_IO_PF_BUDGET_MB (0: auto from the measured rates)
+ double io_budget_frac_ = 0.5; ///< STRATA_IO_PF_BUDGET_FRAC: share of what the drive can read in a window
+ std::vector<IoJob> io_jobs_; ///< (under io_mu_) admitted, not yet started
+ uint64_t io_q_bytes_ = 0; ///< (under io_mu_) their bytes
+ uint64_t io_win_bytes_ = 0; ///< (under io_mu_) admitted in the current window
+ double io_bw_ = 0.0; ///< (under io_mu_) EWMA bytes/us of one worker's reads
+ double io_win_us_ = 0.0; ///< (under io_mu_) EWMA of a decode window's wall time
+ std::chrono::steady_clock::time_point io_win_t0_{};
+ double io_scale_ = 1.0; ///< (under io_mu_) the budget's feedback factor: shrinks while read-ahead goes unused
+ double io_useful_ = -1.0; ///< (under io_mu_) EWMA of used / read-ahead bytes
+ uint64_t io_prev_pf_ = 0, io_prev_used_ = 0; ///< (under io_mu_) the counters at the last window tick
+ double io_target_ = 0.5; ///< STRATA_IO_PF_TARGET: the used share the feedback aims at (0: no feedback)
+ std::atomic<uint64_t> io_epoch_{0}; ///< the decode's layer clock (begin_layer), read by the workers
+ std::unique_ptr<std::atomic<uint8_t>[]> pf_mark_; ///< per (layer, expert): read ahead, not yet asked for (fill mode)
+ std::atomic<uint64_t> io_pf_bytes_{0}, io_pf_used_bytes_{0}, io_pf_late_{0}, io_pf_rej_conf_{0}, io_pf_rej_budget_{0};
+ uint64_t io_budget_now() const; ///< (io_mu_ held) the per-window budget in bytes
+ void io_window_tick(); ///< begin_layer: a new decode window starts at layer 0
std::vector<char> stage_pf_; ///< per stage buffer: filled ahead, not yet used by a layer
std::vector<int> io_fds_; ///< per mapped file: a plain descriptor (POSIX_FADV_RANDOM) for the preads
mutable std::atomic<uint64_t> io_cached_{0}, io_uncached_{0}, io_pread_bytes_{0}, io_pread_us_{0}, io_pread_n_{0};
@@ -983,7 +1100,8 @@ private:
/// `exact` false: four sampled pages (a blob is cached when they are; a handful of syscalls, not hundreds of pages' walk).
uint64_t cached_bytes(int64_t layer, int64_t expert, uint64_t& total, bool exact = true) const;
void io_worker();
- void io_enqueue(int64_t layer, int64_t expert);
+ void io_enqueue(int64_t layer, int64_t expert, bool ahead = false); ///< ahead: a prediction (warm), not a layer's demand
+ static constexpr int32_t kIoAhead = 1 << 30; ///< io_q_: the expert field's flag for a prediction
void io_stop();
const uint8_t* io_blob(int64_t layer, int64_t expert);
const uint8_t* base_ = nullptr;
diff --git a/include/strata/kernels/cpu/pool.hpp b/include/strata/kernels/cpu/pool.hpp
index 686abd6e6..bf0a9bc04 100644
--- a/include/strata/kernels/cpu/pool.hpp
+++ b/include/strata/kernels/cpu/pool.hpp
@@ -61,6 +61,9 @@ struct ExpertJobMulti {
float* out[MAXT] = {};
/// Plan v0.3 P6: a native pack's activations (the layer's `vec_dot_type`), one per token.
const void* nact[MAXT] = {};
+ /// 149-cold: this job's own layout (null: the layer's) - the GGUF read in place, where gate / up / down are three
+ /// tensors: `blob` is the gate rows and the offsets reach the expert's up and down rows in the mapping.
+ const NativeFmt* fmt = nullptr;
};
/// How worker threads are allocated across physical/logical CPU cores (#272). `All` is the layout the pool has
diff --git a/include/strata/prefill/prefill.hpp b/include/strata/prefill/prefill.hpp
index 6c4260824..a34ee6fb5 100644
--- a/include/strata/prefill/prefill.hpp
+++ b/include/strata/prefill/prefill.hpp
@@ -192,6 +192,9 @@ public:
/// says how many of the run's tokens that is (-1 when the run read them all). A read beside the batch slots
/// (STRATA_PIPED_CONCURRENT) gives way to a waiting shorter request there.
std::function<bool()> stop_at_chunk;
+ /// 149-prefetch (STRATA_RAM_ADAPT2): a layer's routed expert ids (T x k, -1 = none) after the prompt path downloaded them
+ /// for its grouping on the host; a policy may count them. Not called on the paths that keep the ids on the GPU.
+ std::function<void(int64_t layer, const int32_t* ids, int64_t n)> on_routes;
int64_t stopped_at() const { return stopped_at_; }
/// Grows this stage's (and the later stages') pinned upload slots to what a text chunk of `T` tokens uploads, so
/// a run of such chunks allocates no pinned memory (STRATA_PIPED_CONCURRENT: an allocation or free on the read's
diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp
index d3fefeeff..a97626a66 100644
--- a/src/core/expert_source.cpp
+++ b/src/core/expert_source.cpp
@@ -653,9 +653,87 @@ FileExpertSource::IoCounters FileExpertSource::io_counters() const {
c.pf_unused = io_pf_unused_.load(std::memory_order_relaxed);
c.crit_us = io_crit_us_.load(std::memory_order_relaxed);
c.crit_n = io_crit_n_.load(std::memory_order_relaxed);
+ c.pf_bytes = io_pf_bytes_.load(std::memory_order_relaxed);
+ c.pf_used_bytes = io_pf_used_bytes_.load(std::memory_order_relaxed);
+ c.pf_late = io_pf_late_.load(std::memory_order_relaxed);
+ c.pf_rej_conf = io_pf_rej_conf_.load(std::memory_order_relaxed);
+ c.pf_rej_budget = io_pf_rej_budget_.load(std::memory_order_relaxed);
+ if (io_sched_) {
+ std::lock_guard<std::mutex> lk(const_cast<std::mutex&>(io_mu_));
+ c.pf_budget_bytes = io_budget_now();
+ }
return c;
}
+// 149-prefetch: the speculative bytes a decode window may read ahead. Fixed (STRATA_IO_PF_BUDGET_MB), or what the workers
+// measured they can read in one window (one worker's rate x the workers x the window's time) times STRATA_IO_PF_BUDGET_FRAC;
+// 256 MB until both rates are known.
+uint64_t FileExpertSource::io_budget_now() const {
+ if (io_budget_fixed_ > 0) return io_budget_fixed_;
+ if (io_bw_ <= 0.0 || io_win_us_ <= 0.0) return 256ull << 20;
+ const double b = io_bw_ * (double) std::max(1, io_threads_) * io_win_us_ * io_budget_frac_ * io_scale_;
+ return (uint64_t) std::max(16.0 * 1048576.0, b);
+}
+
+void FileExpertSource::io_window_tick() {
+ const auto now = std::chrono::steady_clock::now();
+ std::lock_guard<std::mutex> lk(io_mu_);
+ if (io_win_t0_.time_since_epoch().count() != 0) {
+ const double us = std::chrono::duration<double, std::micro>(now - io_win_t0_).count();
+ io_win_us_ = io_win_us_ <= 0.0 ? us : 0.8 * io_win_us_ + 0.2 * us;
+ }
+ io_win_t0_ = now;
+ io_win_bytes_ = 0;
+ // feedback: the share of the read-ahead bytes that a layer then used (the used count lags the reads by a window or
+ // so; an EWMA over windows evens that out). Below STRATA_IO_PF_TARGET the budget shrinks (x0.7 a window, to 1/20),
+ // above it it grows back (x1.1, to 1): a box whose page cache cannot hold what is read ahead stops paying for it.
+ const uint64_t pf = io_pf_bytes_.load(std::memory_order_relaxed), used = io_pf_used_bytes_.load(std::memory_order_relaxed);
+ const uint64_t dpf = pf - io_prev_pf_, du = used - io_prev_used_;
+ io_prev_pf_ = pf;
+ io_prev_used_ = used;
+ if (io_target_ > 0.0 && dpf > 0) {
+ const double u = std::min(1.0, (double) du / (double) dpf);
+ io_useful_ = io_useful_ < 0.0 ? u : 0.8 * io_useful_ + 0.2 * u;
+ io_scale_ = io_useful_ < io_target_ ? std::max(0.05, io_scale_ * 0.7) : std::min(1.0, io_scale_ * 1.1);
+ }
+}
+
+// 149-prefetch: admit the predicted experts that are not resident. An expert's bar is the base confidence, raised with
+// the queue's fill against the window budget (a busy drive takes only the surer guesses); the window's speculative bytes
+// stay within the budget; each job carries the layer clock by which it must have started (else the worker drops it).
+void FileExpertSource::warm_scored(int64_t layer, const int64_t* experts, const float* conf, int64_t n, int ahead) {
+ if (!io_sched_ || !io_pf_ || !direct_.empty() || !io_ahead_) { warm(layer, experts, n); return; }
+ if (n <= 0 || layer < 0 || layer >= n_layers_) return;
+ const uint64_t bytes = layer_blob_bytes_[(size_t) layer];
+ const uint64_t deadline = io_epoch_.load(std::memory_order_relaxed) + (uint64_t) std::max(1, ahead);
+ bool any = false;
+ {
+ std::lock_guard<std::mutex> lk(io_mu_);
+ const uint64_t budget = io_budget_now();
+ for (int64_t j = 0; j < n; ++j) {
+ const int64_t e = experts[j];
+ if (e < 0 || e >= n_expert_) continue;
+ const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) e;
+ if (complement_ready_ && resident_blob(index) != nullptr) continue; // in the RAM copy
+ warm_count_.fetch_add(1, std::memory_order_relaxed);
+ const double pressure = std::min(1.0, (double) io_q_bytes_ / (double) std::max<uint64_t>(1, budget));
+ const double bar = (double) io_minconf_ + (1.0 - (double) io_minconf_) * pressure;
+ const float c = conf != nullptr ? conf[j] : 1.0f;
+ if ((double) c < bar) { io_pf_rej_conf_.fetch_add(1, std::memory_order_relaxed); continue; }
+ if (io_win_bytes_ + bytes > budget) { io_pf_rej_budget_.fetch_add(1, std::memory_order_relaxed); continue; }
+ bool dup = false;
+ for (const IoJob& q : io_jobs_)
+ if (q.layer == (int32_t) layer && q.expert == (int32_t) e) { dup = true; break; }
+ if (dup) continue;
+ io_jobs_.push_back({(int32_t) layer, (int32_t) e, c, deadline, bytes});
+ io_q_bytes_ += bytes;
+ io_win_bytes_ += bytes;
+ any = true;
+ }
+ }
+ if (any) io_cv_.notify_all();
+}
+
void FileExpertSource::set_io_prefetch(bool on, int threads) {
io_stop();
#if defined(__linux__)
@@ -686,6 +764,18 @@ void FileExpertSource::set_io_prefetch(bool on, int threads) {
io_pf_ = true;
io_stats_ = true;
io_quit_ = false;
+ { // 149-prefetch: STRATA_IO_PF_SCHED=1 - deadlines, confidence admission and a per-window byte budget (off: as before)
+ const char* v = std::getenv("STRATA_IO_PF_SCHED");
+ io_sched_ = v != nullptr && std::atoi(v) != 0;
+ if (const char* m = std::getenv("STRATA_IO_PF_MINCONF")) io_minconf_ = (float) std::clamp(std::atof(m), 0.0, 1.0);
+ if (const char* m = std::getenv("STRATA_IO_PF_BUDGET_MB")) io_budget_fixed_ = (uint64_t) std::max(0.0, std::atof(m) * 1048576.0);
+ if (const char* m = std::getenv("STRATA_IO_PF_BUDGET_FRAC")) io_budget_frac_ = std::clamp(std::atof(m), 0.01, 4.0);
+ if (const char* m = std::getenv("STRATA_IO_PF_TARGET")) io_target_ = std::clamp(std::atof(m), 0.0, 1.0);
+ }
+ if (!pf_mark_) {
+ pf_mark_.reset(new std::atomic<uint8_t>[(size_t) (n_layers_ * n_expert_)]);
+ for (int64_t i = 0; i < n_layers_ * n_expert_; ++i) pf_mark_[(size_t) i].store(0, std::memory_order_relaxed);
+ }
for (int i = 0; i < io_threads_; ++i) io_workers_.emplace_back([this] { io_worker(); });
#else
(void) on; (void) threads;
@@ -697,6 +787,8 @@ void FileExpertSource::io_stop() {
std::lock_guard<std::mutex> lk(io_mu_);
io_quit_ = true;
io_q_.clear();
+ io_jobs_.clear();
+ io_q_bytes_ = 0;
}
io_cv_.notify_all();
for (std::thread& t : io_workers_)
@@ -710,14 +802,14 @@ void FileExpertSource::io_stop() {
io_quit_ = false;
}
-void FileExpertSource::io_enqueue(int64_t layer, int64_t expert) {
+void FileExpertSource::io_enqueue(int64_t layer, int64_t expert, bool ahead) {
{
std::lock_guard<std::mutex> lk(io_mu_);
if (io_q_.size() >= 512) { // the predictions run ahead of the reads: the oldest are the stalest
io_q_.pop_front();
io_pf_dropped_.fetch_add(1, std::memory_order_relaxed);
}
- io_q_.emplace_back((int32_t) layer, (int32_t) expert);
+ io_q_.emplace_back((int32_t) layer, (int32_t) expert | (ahead ? kIoAhead : 0));
}
io_cv_.notify_one();
}
@@ -727,24 +819,66 @@ void FileExpertSource::io_enqueue(int64_t layer, int64_t expert) {
void FileExpertSource::io_worker() {
for (;;) {
int32_t layer, expert;
+ bool ahead = true; // a prediction (marked for the used / unused count), not a layer's own demand read
{
std::unique_lock<std::mutex> lk(io_mu_);
- io_cv_.wait(lk, [&] { return io_quit_ || !io_q_.empty(); });
+ io_cv_.wait(lk, [&] { return io_quit_ || !io_q_.empty() || !io_jobs_.empty(); });
if (io_quit_) return;
- layer = io_q_.back().first; // the newest prediction first: it is the layer that comes next
- expert = io_q_.back().second;
- io_q_.pop_back();
+ if (io_sched_ && io_q_.empty() && !io_jobs_.empty()) { // (the layers' own reads in io_q_ go first)
+ // 149-prefetch: drop the jobs whose layer the decode has reached (its demand read covers them); then the
+ // earliest deadline, the surest guess first among equals
+ const uint64_t now = io_epoch_.load(std::memory_order_relaxed);
+ size_t best = io_jobs_.size();
+ for (size_t i = 0; i < io_jobs_.size();) {
+ if (io_jobs_[i].deadline <= now) {
+ io_q_bytes_ -= std::min(io_q_bytes_, io_jobs_[i].bytes);
+ io_jobs_[i] = io_jobs_.back();
+ io_jobs_.pop_back();
+ io_pf_late_.fetch_add(1, std::memory_order_relaxed);
+ continue;
+ }
+ ++i;
+ }
+ best = io_jobs_.size();
+ for (size_t i = 0; i < io_jobs_.size(); ++i)
+ if (best == io_jobs_.size() || io_jobs_[i].deadline < io_jobs_[best].deadline ||
+ (io_jobs_[i].deadline == io_jobs_[best].deadline && io_jobs_[i].conf > io_jobs_[best].conf))
+ best = i;
+ if (best == io_jobs_.size()) continue; // all were late
+ layer = io_jobs_[best].layer;
+ expert = io_jobs_[best].expert;
+ io_q_bytes_ -= std::min(io_q_bytes_, io_jobs_[best].bytes);
+ io_jobs_[best] = io_jobs_.back();
+ io_jobs_.pop_back();
+ } else {
+ if (io_q_.empty()) continue;
+ layer = io_q_.back().first; // the newest prediction first: it is the layer that comes next
+ expert = io_q_.back().second & ~kIoAhead;
+ ahead = (io_q_.back().second & kIoAhead) != 0;
+ io_q_.pop_back();
+ }
}
io_pf_jobs_.fetch_add(1, std::memory_order_relaxed);
const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
if (complement_ready_ && resident_blob(index) != nullptr) continue;
if (!override_.empty() && override_[index] != nullptr) continue;
+ const auto tjob = std::chrono::steady_clock::now();
+ // 149-prefetch: one worker's read rate (bytes per microsecond), for the scheduler's window budget
+ auto note_rate = [&](uint64_t b) {
+ if (b == 0) return;
+ const double us = std::max(1.0, std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - tjob).count());
+ io_pf_bytes_.fetch_add(b, std::memory_order_relaxed);
+ std::lock_guard<std::mutex> lk(io_mu_);
+ const double r = (double) b / us;
+ io_bw_ = io_bw_ <= 0.0 ? r : 0.9 * io_bw_ + 0.1 * r;
+ };
if (io_fill_) { // read what is not cached into a scratch buffer: the pages stay in the page cache for the kernels
#if defined(__linux__)
static thread_local std::vector<uint8_t> scratch;
Span sp[3];
const int ns = spans(layer, expert, sp);
bool any = false;
+ uint64_t got = 0;
for (int i = 0; i < ns; ++i) {
if (sp[i].fd < 0 || resident_sampled_n(sp[i].p, sp[i].n)) continue;
if (scratch.size() < sp[i].n) scratch.resize((size_t) sp[i].n);
@@ -755,9 +889,14 @@ void FileExpertSource::io_worker() {
std::memory_order_relaxed);
io_pread_n_.fetch_add(1, std::memory_order_relaxed);
any = true;
+ got += sp[i].n;
}
}
- if (any) io_pf_blobs_.fetch_add(1, std::memory_order_relaxed);
+ if (any) {
+ io_pf_blobs_.fetch_add(1, std::memory_order_relaxed);
+ if (pf_mark_ && ahead) pf_mark_[index].store(1, std::memory_order_relaxed); // read ahead: scored at the demand
+ note_rate(got);
+ }
else io_pf_skips_.fetch_add(1, std::memory_order_relaxed);
#endif
continue;
@@ -781,6 +920,7 @@ void FileExpertSource::io_worker() {
const bool ok = copy_from_files(layer, expert, dst);
io_pf_blobs_.fetch_add(1, std::memory_order_relaxed);
if (ok) file_read_bytes_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);
+ if (ok) note_rate(layer_blob_bytes_[(size_t) layer]);
{
std::lock_guard<std::mutex> lk(stage_mu_);
stage_busy_[v] = 0;
@@ -998,6 +1138,8 @@ bool FileExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64
}
void FileExpertSource::close() {
+ prefetch_end();
+ ram_stop();
io_stop();
inputs_.clear();
if (complement_arena_ != nullptr) {
@@ -1281,7 +1423,11 @@ bool FileExpertSource::claim_stage(int64_t key, size_t& v, bool& fill, bool ahea
v = it->second;
stage_epoch_[v] = epoch_;
stage_used_[v] = seq;
- if (!ahead && stage_pf_[v]) { stage_pf_[v] = 0; io_pf_used_.fetch_add(1, std::memory_order_relaxed); } // a layer asked for what was read ahead
+ if (!ahead && stage_pf_[v]) { // a layer asked for what was read ahead
+ stage_pf_[v] = 0;
+ io_pf_used_.fetch_add(1, std::memory_order_relaxed);
+ io_pf_used_bytes_.fetch_add(layer_blob_bytes_[(size_t) (key / n_expert_)], std::memory_order_relaxed);
+ }
return true;
}
v = stage_buf_.size();
@@ -1455,6 +1601,10 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
const int64_t e = experts[i];
if (e < 0 || e >= n_expert_) continue;
const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) e;
+ if (pf_mark_ && pf_mark_[index].exchange(0, std::memory_order_relaxed)) { // 149-prefetch: read ahead, now asked for
+ io_pf_used_.fetch_add(1, std::memory_order_relaxed);
+ io_pf_used_bytes_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);
+ }
if (complement_ready_ && resident_blob(index) != nullptr) continue;
if (!override_.empty() && override_[index] != nullptr) continue;
uint64_t total = 0;
@@ -1465,6 +1615,33 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
return;
}
std::vector<Fill> todo;
+ const auto t0 = std::chrono::steady_clock::now();
+ claim_todo(layer, experts, n, todo, nullptr);
+ fill_many(todo);
+ if (io_pf_ && !todo.empty()) {
+ io_crit_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),
+ std::memory_order_relaxed);
+ io_crit_n_.fetch_add(1, std::memory_order_relaxed);
+ }
+}
+
+void FileExpertSource::claim_todo(int64_t layer, const int64_t* experts, int64_t n, std::vector<Fill>& todo, uint8_t* pending,
+ bool zc) {
+ // 149-cold mode 3: the GGUF in place, mapped and buffered: an expert the page cache holds whole is computed from
+ // the mapping (its three tensors' rows), not assembled into a stage buffer first. Decided here, before the lock.
+ zc_keys_.clear();
+ std::vector<char> zc_ok;
+ if (zc && !role_ptr_.empty() && !unmapped_ && direct_.empty() && strata::kernels::cpu::expert_layout().native &&
+ strata::kernels::cpu::expert_layout().fmt[(size_t) layer].tail_off == 0) {
+ zc_ok.assign((size_t) n, 0);
+ for (int64_t i = 0; i < n; ++i) {
+ const int64_t e = experts[i];
+ if (e < 0 || e >= n_expert_) continue;
+ uint64_t total = 0;
+ const uint64_t c = cached_bytes(layer, e, total, true);
+ zc_ok[(size_t) i] = total > 0 && c >= total ? 1 : 0; // no residency query (not Linux): never
+ }
+ }
// experts.bin with io prefetch: a blob whose pages are cached is handed out as the mapping (io_blob); only the
// uncached ones are read here, together, on the fetch threads
const bool only_uncached = io_pf_ && role_ptr_.empty() && direct_.empty();
@@ -1478,7 +1655,6 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
cold[(size_t) i] = cached_bytes(layer, e, total, false) < total ? 1 : 0;
}
}
- const auto t0 = std::chrono::steady_clock::now();
{
std::lock_guard<std::mutex> lk(stage_mu_);
for (int64_t i = 0; i < n; ++i) {
@@ -1489,23 +1665,131 @@ void FileExpertSource::prefetch(int64_t layer, const int64_t* experts, int64_t n
continue; // in the RAM copy
if (!override_.empty() && override_[index] != nullptr) continue;
if (only_uncached && !cold[(size_t) i] && stage_of_.find(layer * n_expert_ + e) == stage_of_.end()) continue;
+ if (!zc_ok.empty() && zc_ok[(size_t) i] && stage_of_.find(layer * n_expert_ + e) == stage_of_.end()) {
+ zc_keys_.push_back(layer * n_expert_ + e); // read from the mapping by the kernels themselves
+ continue;
+ }
size_t v = 0;
bool fill = false;
if (!claim_stage(layer * n_expert_ + e, v, fill) && fill) {
todo.push_back({v, layer, e, stage_buf_[v].get()});
+ if (pending != nullptr) pending[i] = 1;
if (warm_stamp_) {
const uint32_t s = warm_stamp_[index].load(std::memory_order_relaxed);
if (s != 0 && (uint64_t) s + 3 >= epoch_ + 1) warm_hits_.fetch_add(1, std::memory_order_relaxed);
}
+ } else if (pending != nullptr && fill == false) {
+ // being filled by another thread (a read ahead, the adaptive tier): not ready yet either
+ const auto it = stage_of_.find(layer * n_expert_ + e);
+ if (it != stage_of_.end() && stage_busy_[it->second]) pending[i] = 1;
}
}
}
- fill_many(todo);
- if (io_pf_ && !todo.empty()) {
- io_crit_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),
- std::memory_order_relaxed);
+}
+
+// 149-cold STRATA_COLD_OVERLAP=1: the claimed blobs are read on background threads (the decode thread computes the
+// ready experts meanwhile); the same reads as fill_many, on as many threads, each blob published as it lands.
+int64_t FileExpertSource::prefetch_begin(int64_t layer, const int64_t* experts, int64_t n, uint8_t* pending) {
+ for (int64_t i = 0; i < n; ++i) pending[i] = 0;
+ if (!staged() || n <= 0 || layer < 0 || layer >= n_layers_) return 0;
+ if (io_fill_ && io_pf_ && direct_.empty()) { // the page-fill mode: nothing is staged here, the kernels wait on pages
+ prefetch(layer, experts, n);
+ return 0;
+ }
+ prefetch_end();
+ async_todo_.clear();
+ async_fill_us_ = 0;
+ static const int mode = [] { const char* v = std::getenv("STRATA_COLD_OVERLAP"); return v != nullptr ? std::atoi(v) : 0; }();
+ claim_todo(layer, experts, n, async_todo_, pending, mode >= 3);
+ if (mode == 4) { // zero-copy only: the blobs that must be read are read here, as `prefetch` reads them
+ const auto t0 = std::chrono::steady_clock::now();
+ fill_many(async_todo_);
+ if (io_pf_ && !async_todo_.empty()) {
+ io_crit_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),
+ std::memory_order_relaxed);
+ io_crit_n_.fetch_add(1, std::memory_order_relaxed);
+ }
+ async_todo_.clear();
+ for (int64_t i = 0; i < n; ++i) pending[i] = 0; // one another thread still fills: `blob` waits for it
+ return 0;
+ }
+ int64_t np = 0;
+ for (int64_t i = 0; i < n; ++i) np += pending[i] ? 1 : 0;
+ if (async_todo_.empty()) return np;
+ async_t0_ = std::chrono::steady_clock::now();
+ async_next_.store(0, std::memory_order_relaxed);
+ async_end_ns_.store(0, std::memory_order_relaxed);
+ const bool direct = !direct_.empty();
+ auto work = [this, direct] {
+ const std::vector<Fill>& todo = async_todo_;
+ if (direct) {
+ for (size_t at; (at = async_next_.fetch_add(16)) < todo.size();) {
+ const size_t k = std::min<size_t>(16, todo.size() - at);
+ const auto t0 = std::chrono::steady_clock::now();
+ const bool ok = read_direct(todo.data() + at, k);
+ const double us = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count();
+ for (size_t i = at; i < at + k; ++i) {
+ const Fill& f = todo[i];
+ if (ok) publish_stage(f.v, f.layer, true, us / (double) k);
+ else (void) fill_stage(f.v, f.layer, f.e, f.dst);
+ }
+ }
+ } else {
+ for (size_t i; (i = async_next_.fetch_add(1)) < todo.size();)
+ (void) fill_stage(todo[i].v, todo[i].layer, todo[i].e, todo[i].dst);
+ }
+ const int64_t ns = (int64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(
+ std::chrono::steady_clock::now() - async_t0_).count();
+ int64_t cur = async_end_ns_.load(std::memory_order_relaxed);
+ while (ns > cur && !async_end_ns_.compare_exchange_weak(cur, ns, std::memory_order_relaxed)) {}
+ };
+ const size_t nt = direct ? std::min<size_t>(4, (async_todo_.size() + 15) / 16)
+ : std::min<size_t>(async_todo_.size(), (size_t) std::max(1, fetch_threads_));
+ for (size_t t = 0; t < nt; ++t) async_th_.emplace_back(work);
+ return np;
+}
+
+int64_t FileExpertSource::wait_landed(int64_t layer, const int64_t* experts, int64_t n, uint8_t* state) {
+ std::unique_lock<std::mutex> lk(stage_mu_);
+ int64_t got = 0;
+ stage_cv_.wait(lk, [&] {
+ bool left = false;
+ for (int64_t i = 0; i < n; ++i) {
+ if (state[i] != 1) continue;
+ const auto it = stage_of_.find(layer * n_expert_ + experts[i]);
+ // published (or failed: then `blob` reads it again on this thread, as before)
+ if (it == stage_of_.end() || !stage_busy_[it->second]) { state[i] = 2; ++got; }
+ else left = true;
+ }
+ return got > 0 || !left;
+ });
+ return got;
+}
+
+bool FileExpertSource::zc_roles(int64_t layer, int64_t expert, const uint8_t** p) {
+ const int64_t key = layer * n_expert_ + expert;
+ if (std::find(zc_keys_.begin(), zc_keys_.end(), key) == zc_keys_.end()) return false;
+ for (int r = 0; r < 3; ++r) {
+ const size_t i = (size_t) (3 * layer + r);
+ p[r] = role_ptr_[i] + (size_t) ((uint64_t) expert * role_bytes_[i]);
+ }
+ file_read_bytes_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);
+ if (complement_ready_) file_reads_.fetch_add(1, std::memory_order_relaxed);
+ reads_.fetch_add(1, std::memory_order_relaxed);
+ if (io_stats_) io_cached_.fetch_add(layer_blob_bytes_[(size_t) layer], std::memory_order_relaxed);
+ return true;
+}
+
+void FileExpertSource::prefetch_end() {
+ if (async_th_.empty()) return;
+ for (std::thread& t : async_th_) t.join();
+ async_th_.clear();
+ async_fill_us_ = (double) async_end_ns_.load(std::memory_order_relaxed) / 1e3;
+ if (io_pf_ && !async_todo_.empty()) { // the reads' wall time, as the synchronous fill counts it
+ io_crit_us_.fetch_add((uint64_t) async_fill_us_, std::memory_order_relaxed);
io_crit_n_.fetch_add(1, std::memory_order_relaxed);
}
+ async_todo_.clear();
}
void FileExpertSource::prefetch_pairs(const std::pair<int32_t, int32_t>* pairs, int64_t n) {
@@ -2181,7 +2465,7 @@ void FileExpertSource::warm(int64_t layer, const int64_t* experts, int64_t n) {
if (warm_stamp_) warm_stamp_[index].store(stamp, std::memory_order_relaxed);
warm_count_.fetch_add(1, std::memory_order_relaxed);
if (io_pf_ && direct_.empty()) { // read it ahead with pread on the I/O threads (instead of an madvise hint)
- if (io_ahead_) io_enqueue(layer, e);
+ if (io_ahead_) io_enqueue(layer, e, true);
continue;
}
if (role_ptr_.empty() || unmapped_) continue;
@@ -2360,6 +2644,7 @@ void RouterLookahead::run() {
std::vector<float> logits((size_t) (8 * n_expert_));
std::vector<int32_t> order((size_t) n_expert_);
std::vector<int64_t> want;
+ std::vector<float> wconf; // 149-prefetch: per wanted expert, its best rank's confidence (warm_scored)
for (;;) {
int64_t layer, nt;
const int32_t* host_res;
@@ -2382,6 +2667,8 @@ void RouterLookahead::run() {
for (int dj = 0; dj < depth_ && layer0 + dj < (int64_t) routers_.size(); ++dj) {
layer = layer0 + dj;
want.clear();
+ wconf.clear();
+ const bool scored = src_->wants_scores();
// The router dot is AVX2 (kq_avx2.cpp); a CPU without AVX2 (the experimental older-CPU builds) takes the AVX1
// one (kq_avx1.cpp) or, without AVX, the same sums in plain C++. From the Strata_Dirigo fork (rwkeyes): an
// AVX-only Xeon E5-2687W died here (vpmovzxwd) on its first request. An estimate only (which experts to
@@ -2423,7 +2710,14 @@ void RouterLookahead::run() {
for (int j = 0; j < kmax; ++j) {
const int64_t e = order[(size_t) j];
if (host_res != nullptr && host_res[(size_t) (layer * n_expert_ + e)] >= 0) continue; // on the GPU
- if (j < k_ && std::find(want.begin(), want.end(), e) == want.end()) want.push_back(e);
+ if (j < k_) {
+ const auto it = std::find(want.begin(), want.end(), e);
+ // confidence: 1 for a token's top pick down to 1/k_ for its k_-th, the best over the window's tokens,
+ // halved for every layer further ahead than the next (a deeper guess is less often right)
+ const float c = (float) (k_ - j) / (float) k_ * (dj == 0 ? 1.0f : std::pow(0.5f, (float) dj));
+ if (it == want.end()) { want.push_back(e); wconf.push_back(c); }
+ else if (scored && c > wconf[(size_t) (it - want.begin())]) wconf[(size_t) (it - want.begin())] = c;
+ }
if (stats_on_ && dj == 0)
for (int s = 0; s < 3; ++s)
if (j < ks[s]) sm[(size_t) (s * words_ + (e >> 6))] |= 1ull << (e & 63);
@@ -2453,7 +2747,8 @@ void RouterLookahead::run() {
pmask_[(size_t) (layer * 3 * words_ + i)].store(sm[(size_t) i], std::memory_order_relaxed);
pstamp_[(size_t) layer].store(sgen + 1, std::memory_order_release);
}
- src_->warm(layer, want.data(), (int64_t) want.size());
+ if (scored) src_->warm_scored(layer, want.data(), wconf.data(), (int64_t) want.size(), dj + 1);
+ else src_->warm(layer, want.data(), (int64_t) want.size());
if (fs != nullptr && dj == 0) // the next layer only: a deeper guess is too often wrong to spend a copy on
for (const int64_t e : want) fs->predict(src_, layer, (int32_t) e);
predicted_.fetch_add((int64_t) want.size(), std::memory_order_relaxed);
@@ -2471,11 +2766,17 @@ void FileExpertSource::begin_layer(int64_t layer, const int32_t* ids, int64_t k)
(void) ids;
(void) k;
if (!staged()) return;
- std::lock_guard<std::mutex> lk(stage_mu_);
- if (layer != last_layer_) {
- ++epoch_;
- last_layer_ = layer;
+ bool new_window = false;
+ {
+ std::lock_guard<std::mutex> lk(stage_mu_);
+ if (layer != last_layer_) {
+ ++epoch_;
+ new_window = layer < last_layer_;
+ last_layer_ = layer;
+ io_epoch_.store(epoch_, std::memory_order_relaxed); // 149-prefetch: the workers' deadline clock
+ }
}
+ if (new_window && io_sched_) io_window_tick();
}
bool FileExpertSource::transient(int64_t layer, int64_t expert) const {
@@ -3465,6 +3766,7 @@ bool FileExpertSource::replace_resident(int64_t layer, int64_t in, int64_t out)
const uint64_t at = complement_offsets_[i_out], bytes = layer_blob_bytes_[(size_t) layer];
if (at == kNoComplement || complement_offsets_[i_in] != kNoComplement || at > complement_bytes_ ||
bytes > complement_bytes_ - at) return false;
+ (void) at;
// read beside the copy first: a failed read leaves `out` where it was
if (ram_adapt_buf_.size() < (size_t) bytes) ram_adapt_buf_.resize((size_t) bytes);
const auto t0 = std::chrono::steady_clock::now();
@@ -3472,10 +3774,26 @@ bool FileExpertSource::replace_resident(int64_t layer, int64_t in, int64_t out)
file_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),
std::memory_order_relaxed);
file_read_bytes_.fetch_add(bytes, std::memory_order_relaxed);
+ return install_resident(layer, in, out, ram_adapt_buf_.data());
+}
+
+// `in`'s bytes (src) into `out`'s place of the compact copy, and the two offsets swapped - the checks again, as the caller
+// may have read `src` a while ago (149-prefetch: ram_install)
+bool FileExpertSource::install_resident(int64_t layer, int64_t in, int64_t out, const uint8_t* src) {
+ if (!complement_ready_ || complement_host_ == nullptr || exchange_storage_.active() || !staged_.empty() ||
+ layer < 0 || layer >= n_layers_ || in < 0 || out < 0 || in >= n_expert_ || out >= n_expert_ || in == out || src == nullptr)
+ return false;
+ const size_t i_in = (size_t) layer * (size_t) n_expert_ + (size_t) in;
+ const size_t i_out = (size_t) layer * (size_t) n_expert_ + (size_t) out;
+ if (i_in >= complement_offsets_.size() || i_out >= complement_offsets_.size()) return false;
+ if (!override_.empty() && (override_[i_in] != nullptr || override_[i_out] != nullptr)) return false;
+ const uint64_t at = complement_offsets_[i_out], bytes = layer_blob_bytes_[(size_t) layer];
+ if (at == kNoComplement || complement_offsets_[i_in] != kNoComplement || at > complement_bytes_ ||
+ bytes > complement_bytes_ - at) return false;
// #2134: a Foresight slot may be copying out of `out`'s place right now; it is dropped (never served) before the place is rewritten
ForesightSwap* fs = fs_swap_.load(std::memory_order_acquire);
if (fs != nullptr) fs->hold(layer, (int32_t) out);
- std::memcpy((uint8_t*) complement_host_ + (size_t) at, ram_adapt_buf_.data(), (size_t) bytes);
+ std::memcpy((uint8_t*) complement_host_ + (size_t) at, src, (size_t) bytes);
const bool moved = detail::exchange_cache_complement(complement_offsets_, i_out, i_in);
if (fs != nullptr) fs->unhold(layer, (int32_t) out);
if (!moved) return false;
@@ -3483,6 +3801,119 @@ bool FileExpertSource::replace_resident(int64_t layer, int64_t in, int64_t out)
return true;
}
+// ---- 149-prefetch: staged promotions. The read of `in` runs on this source's own thread (one at a time: the drive's
+// queue stays the decode's first); the install is the decode thread's (ram_install, at a window boundary).
+bool FileExpertSource::ram_stage(int64_t layer, int64_t in, int64_t out) {
+ if (!complement_ready_ || complement_host_ == nullptr || layer < 0 || layer >= n_layers_ || in < 0 || out < 0 ||
+ in >= n_expert_ || out >= n_expert_ || in == out)
+ return false;
+ {
+ std::lock_guard<std::mutex> lk(ram_mu_);
+ static const size_t cap = [] {
+ const char* v = std::getenv("STRATA_RAM_ADAPT_QUEUE");
+ return v != nullptr ? (size_t) std::max(1, std::atoi(v)) : (size_t) 64;
+ }();
+ if (ram_jobs_.size() >= cap) return false;
+ for (const auto& j : ram_jobs_)
+ if (j->layer == layer && (j->in == in || j->out == out || j->in == out || j->out == in)) return false;
+ auto j = std::make_unique<RamJob>();
+ j->layer = layer; j->in = in; j->out = out; j->state = 0;
+ ram_jobs_.push_back(std::move(j));
+ if (!ram_thread_.joinable()) {
+ ram_quit_ = false;
+ ram_thread_ = std::thread([this] { ram_worker(); });
+ }
+ }
+ ram_cv_.notify_one();
+ return true;
+}
+
+bool FileExpertSource::ram_queued(int64_t layer, int64_t expert) const {
+ std::lock_guard<std::mutex> lk(ram_mu_);
+ for (const auto& j : ram_jobs_)
+ if (j->layer == layer && (j->in == expert || j->out == expert)) return true;
+ return false;
+}
+
+void FileExpertSource::ram_worker() {
+ for (;;) {
+ RamJob* job = nullptr;
+ {
+ std::unique_lock<std::mutex> lk(ram_mu_);
+ ram_cv_.wait(lk, [&] {
+ if (ram_quit_) return true;
+ for (const auto& j : ram_jobs_)
+ if (j->state == 0) return true;
+ return false;
+ });
+ if (ram_quit_) return;
+ for (const auto& j : ram_jobs_)
+ if (j->state == 0) { job = j.get(); break; }
+ job->state = 1;
+ }
+ const uint64_t bytes = layer_blob_bytes_[(size_t) job->layer];
+ job->buf.resize((size_t) bytes);
+ const auto t0 = std::chrono::steady_clock::now();
+ const bool ok = copy_from_files(job->layer, job->in, job->buf.data());
+ file_us_.fetch_add((uint64_t) std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - t0).count(),
+ std::memory_order_relaxed);
+ if (ok) {
+ file_read_bytes_.fetch_add(bytes, std::memory_order_relaxed);
+ ram_staged_reads_.fetch_add(1, std::memory_order_relaxed);
+ }
+ std::lock_guard<std::mutex> lk(ram_mu_);
+ job->state = ok ? 2 : 3;
+ }
+}
+
+int64_t FileExpertSource::ram_install(const int32_t* host_res, int64_t n_expert) {
+ std::vector<std::unique_ptr<RamJob>> done;
+ {
+ std::lock_guard<std::mutex> lk(ram_mu_);
+ for (auto it = ram_jobs_.begin(); it != ram_jobs_.end();) {
+ if ((*it)->state >= 2) { done.push_back(std::move(*it)); it = ram_jobs_.erase(it); }
+ else ++it;
+ }
+ }
+ int64_t n = 0;
+ for (const auto& j : done) {
+ if (j->state != 2) { ram_install_skips_.fetch_add(1, std::memory_order_relaxed); continue; }
+ if (host_res != nullptr && (host_res[(size_t) (j->layer * n_expert + j->in)] >= 0 ||
+ host_res[(size_t) (j->layer * n_expert + j->out)] >= 0)) {
+ ram_install_gpu_.fetch_add(1, std::memory_order_relaxed); // the GPU tier took one of them meanwhile
+ continue;
+ }
+ if (install_resident(j->layer, j->in, j->out, j->buf.data())) ++n;
+ else ram_install_skips_.fetch_add(1, std::memory_order_relaxed);
+ }
+ return n;
+}
+
+void FileExpertSource::ram_stop() {
+ {
+ std::lock_guard<std::mutex> lk(ram_mu_);
+ ram_quit_ = true;
+ }
+ ram_cv_.notify_all();
+ if (ram_thread_.joinable()) ram_thread_.join();
+ std::lock_guard<std::mutex> lk(ram_mu_);
+ ram_jobs_.clear();
+ ram_quit_ = false;
+}
+
+int FileExpertSource::tier_of(int64_t layer, int64_t expert) const {
+ if (layer < 0 || expert < 0 || layer >= n_layers_ || expert >= n_expert_) return 2;
+ const size_t index = (size_t) layer * (size_t) n_expert_ + (size_t) expert;
+ if (complement_ready_ && resident_blob(index) != nullptr) return 0;
+#if defined(__linux__)
+ uint64_t total = 0;
+ const uint64_t c = cached_bytes(layer, expert, total, false);
+ return total > 0 && c >= total ? 1 : 2;
+#else
+ return 2;
+#endif
+}
+
void FileExpertSource::commit_copies() {
fs_hold_staged(true); // #2134: held until commit_flip has pointed `out` at its new place
if (exchange_storage_.active()) return; // STRATA_EXCHANGE_ROTATE: ownership moves in commit_flip, nothing to copy
@@ -3814,6 +4245,12 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
if (ptrace) { std::fprintf(stderr, "pool trace: layer %lld %s %lld\n", (long long) d.layers, what, a); std::fflush(stderr); }
};
const auto c0 = std::chrono::steady_clock::now();
+ // 149-cold STRATA_COLD_OVERLAP: 0 off (default), 1 overlap, 2 measure only, 3 overlap + zero-copy, 4 zero-copy only
+ static const int cold_env = [] {
+ const char* v = std::getenv("STRATA_COLD_OVERLAP");
+ return v == nullptr ? 0 : std::clamp(std::atoi(v), 0, 4);
+ }();
+ d.cold_mode = cold_env;
pt("begin");
d.src->begin_layer(d.layers, ids, n_tok * k);
pt("begun");
@@ -4012,21 +4449,56 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
}
}
const auto c2 = std::chrono::steady_clock::now();
+ // 149-cold STRATA_COLD_OVERLAP (d.cold_mode): which of the CPU's experts still have a blob to read. Mode 1 reads
+ // them in the background and computes the ready ones first, then each read one as it lands; every expert keeps
+ // its own token rows (the pool splits rows, never a row's sum), and the weighted sum over a token's experts is
+ // the GPU's, after the dispatch - so the result is the same bits in any batch order. The stage buffers stay
+ // claimed (busy, then young) until the dispatch has joined its reads and run every batch.
+ static thread_local std::vector<int64_t> miss;
+ static thread_local std::vector<uint8_t> mstate; // per miss: 0 ready, 1 being read, 2 landed, 3 computed
+ miss.clear();
+ int64_t npend = 0;
+ double cold_r_ms = 0;
if (any_cpu) { // CS-T: the experts the CPU computes, fetched together (the GGUF in place reads them on several threads)
- static thread_local std::vector<int64_t> miss;
- miss.clear();
for (int64_t i = 0; i < n_tok * k; ++i)
if (kind[i] == -1 && ids[i] >= 0 && ids[i] < d.n_expert &&
std::find(miss.begin(), miss.end(), (int64_t) ids[i]) == miss.end())
miss.push_back(ids[i]);
- d.src->prefetch(d.layers, miss.data(), (int64_t) miss.size());
+ if (d.cold_mode == 0) {
+ d.src->prefetch(d.layers, miss.data(), (int64_t) miss.size());
+ } else {
+ mstate.assign(miss.size(), 0);
+ const auto r0 = std::chrono::steady_clock::now();
+ npend = d.src->prefetch_begin(d.layers, miss.data(), (int64_t) miss.size(), mstate.data());
+ if (npend > 0 && d.cold_mode == 2) { // measure: every read first, as mode 0 does
+ d.src->prefetch_end();
+ for (uint8_t& s : mstate) if (s == 1) s = 2;
+ cold_r_ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - r0).count();
+ d.ms_cold_wait += cold_r_ms;
+ }
+ }
}
- int njobs = 0;
+ // the CPU's experts this batch takes: all (mode 0, or no read), else by their state (want 0: the ready ones,
+ // want 2: those that landed since the last batch)
+ auto take = [&](int64_t e, uint8_t want) {
+ if (npend == 0) return true;
+ for (size_t q = 0; q < miss.size(); ++q)
+ if (miss[q] == e) return mstate[q] == want;
+ return want == 0;
+ };
+ int njobs = 0, njobs_all = 0;
+ bool first_pass = true;
+ uint8_t want = 0;
+ double run_ms_extra = 0, cr_ms = 0;
+ for (;;) {
for (int64_t t = 0; t < n_tok; ++t)
for (int64_t j = 0; j < k; ++j) {
const int64_t i = t * k + j;
const int64_t e = ids[i];
float* row = out + (size_t) i * H;
+ if (!first_pass && (e < 0 || e >= d.n_expert || kind[i] != -1 || !take(e, want)))
+ continue; // a later batch: only the CPU entries of the experts that just landed
+ if (first_pass && kind[i] == -1 && e >= 0 && e < d.n_expert && !take(e, 0)) continue; // being read: a later batch
if (e < 0 || e >= d.n_expert) {
d.failed = true;
d.fail = "a routed expert id is out of range";
@@ -4053,7 +4525,9 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
++d.cache_refused;
int16_t& jo = d.job_of[(size_t) e];
if (jo < 0) {
- const uint8_t* b = d.src->blob(d.layers, e);
+ const uint8_t* zp[3] = {nullptr, nullptr, nullptr};
+ const bool zc = d.cold_mode >= 3 && native && d.src->zc_roles(d.layers, e, zp);
+ const uint8_t* b = zc ? zp[0] : d.src->blob(d.layers, e);
if (b == nullptr) {
d.failed = true;
d.fail = "the expert source could not produce a blob";
@@ -4066,6 +4540,16 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
ExpertJobMulti& nj = d.jobs_multi[(size_t) jo];
nj.blob = b;
nj.nt = 0;
+ nj.fmt = nullptr;
+ if (zc) { // 149-cold mode 3: the gate rows are the blob; up and down at their distance in the mapping
+ if (d.zc_fmt.size() < d.jobs_multi.size()) d.zc_fmt.resize(d.jobs_multi.size());
+ strata::kernels::cpu::NativeFmt& f = d.zc_fmt[(size_t) jo];
+ f = lay.fmt[(size_t) d.layers];
+ f.up_off = (size_t) ((uintptr_t) zp[1] - (uintptr_t) zp[0]); // modular: the tensors' order
+ f.down_off = (size_t) ((uintptr_t) zp[2] - (uintptr_t) zp[0]); // in the file does not matter
+ nj.fmt = &f;
+ ++d.cold_zc;
+ }
}
ExpertJobMulti& jb = d.jobs_multi[(size_t) jo];
jb.act[jb.nt] = &d.act_multi[(size_t) t];
@@ -4074,10 +4558,56 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
++jb.nt;
++d.multi_entries;
}
+ if (npend == 0) break;
+ // 149-cold: run this batch now (the ready experts, then each landed group), then wait for the next landing
+ {
+ const auto b0 = std::chrono::steady_clock::now();
+ if (njobs > 0) {
+ if (native) d.pool->run_split_multi_native(lay.fmt[(size_t) d.layers], d.jobs_multi.data(), njobs);
+ else d.pool->run_split_multi(d.jobs_multi.data(), njobs);
+ ++d.cold_batches;
+ }
+ const auto b1 = std::chrono::steady_clock::now();
+ const double bm = std::chrono::duration<double, std::milli>(b1 - b0).count();
+ run_ms_extra += bm;
+ if (first_pass) { d.ms_cold_cr += bm; cr_ms = bm; }
+ else d.ms_cold_cc += bm;
+ if (first_pass) d.cold_ready += njobs;
+ // the computed ones' job slots are free again (an expert is in one batch only)
+ for (int jx = 0; jx < njobs; ++jx) d.jobs_multi[(size_t) jx].nt = 0;
+ for (int64_t i = 0; i < n_tok * k; ++i)
+ if (ids[i] >= 0 && ids[i] < d.n_expert) d.job_of[(size_t) ids[i]] = -1;
+ if (!first_pass) for (uint8_t& s : mstate) if (s == 2) s = 3;
+ njobs_all += njobs;
+ njobs = 0;
+ first_pass = false;
+ want = 2;
+ bool left = false;
+ for (uint8_t s : mstate) left = left || s == 1 || s == 2;
+ if (!left) break;
+ if (std::none_of(mstate.begin(), mstate.end(), [](uint8_t s) { return s == 2; })) {
+ const auto w0 = std::chrono::steady_clock::now();
+ (void) d.src->wait_landed(d.layers, miss.data(), (int64_t) miss.size(), mstate.data());
+ d.ms_cold_wait += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - w0).count();
+ }
+ }
+ }
+ if (npend > 0) {
+ d.src->prefetch_end();
+ const FileExpertSource* fs = dynamic_cast<const FileExpertSource*>(d.src);
+ const double r_ms = d.cold_mode == 2 ? cold_r_ms : (fs != nullptr ? fs->async_fill_us() / 1e3 : 0.0);
+ ++d.cold_layers;
+ d.cold_pending += npend;
+ d.ms_cold_r += r_ms;
+ d.ms_cold_hide += std::min(r_ms, cr_ms); // what an overlap can hide of this layer's reads at most
+ njobs = njobs_all;
+ d.ms_run += run_ms_extra; // the batches ran inside the job loop: the pool's time stays the pool's
+ d.ms_jobs -= run_ms_extra;
+ }
const auto c3 = std::chrono::steady_clock::now();
if (d.plan != nullptr) d.plan->cpu_jobs = njobs;
pt("run", njobs);
- if (njobs > 0) {
+ if (njobs > 0 && npend == 0) {
if (native) d.pool->run_split_multi_native(lay.fmt[(size_t) d.layers], d.jobs_multi.data(), njobs);
else d.pool->run_split_multi(d.jobs_multi.data(), njobs);
}
diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cpp
index 9641ba6a9..d9961938a 100644
--- a/src/kernels/cpu/pool.cpp
+++ b/src/kernels/cpu/pool.cpp
@@ -714,32 +714,33 @@ void ExpertPool::drain(int id, ExpertScratch& scratch, uint32_t epoch) {
const int e = (int) (r / per), r0 = (int) (r % per);
const int r1 = (int) std::min<int64_t>(per, r0 + (g1 - r));
SplitBufMulti& sb = split_multi_[(size_t) e];
- if (mode_ == 5 && q2_native_kernels(nfmt_->gu_type)) {
+ const NativeFmt* const nfmt = mjobs_[e].fmt != nullptr ? mjobs_[e].fmt : nfmt_; // 149-cold: same types and rows
+ if (mode_ == 5 && q2_native_kernels(nfmt->gu_type)) {
// a native Q2_0 pack: gate and up rows on the Q2_0 kernels, then SwiGLU
thread_local float gbuf[MAXT][FF], ubuf[MAXT][FF];
float* gp[MAXT];
float* up[MAXT];
for (int t = 0; t < mjobs_[e].nt; ++t) { gp[t] = gbuf[t]; up[t] = ubuf[t]; }
- const int nbk = (int) (nfmt_->n_embd / 64);
- q2_rows_any(mjobs_[e].blob, nfmt_->gu_row, nbk, mjobs_[e].act, mjobs_[e].nt, gp, r0, r1);
- q2_rows_any(mjobs_[e].blob + nfmt_->up_off, nfmt_->gu_row, nbk, mjobs_[e].act, mjobs_[e].nt, up, r0, r1);
+ const int nbk = (int) (nfmt->n_embd / 64);
+ q2_rows_any(mjobs_[e].blob, nfmt->gu_row, nbk, mjobs_[e].act, mjobs_[e].nt, gp, r0, r1);
+ q2_rows_any(mjobs_[e].blob + nfmt->up_off, nfmt->gu_row, nbk, mjobs_[e].act, mjobs_[e].nt, up, r0, r1);
for (int t = 0; t < mjobs_[e].nt; ++t)
for (int r = r0; r < r1; ++r)
sb.ff[t][r] = (gbuf[t][r] / (1.f + std::exp(-gbuf[t][r]))) * ubuf[t][r];
} else if (mode_ == 5) {
float* ff[MAXT];
for (int t = 0; t < mjobs_[e].nt; ++t) ff[t] = sb.ff[t];
- native_gu_rows(*nfmt_, mjobs_[e].blob, mjobs_[e].nact, mjobs_[e].nt, ff, r0, r1);
- } else if (q2_native_kernels(nfmt_->d_type)) {
+ native_gu_rows(*nfmt, mjobs_[e].blob, mjobs_[e].nact, mjobs_[e].nt, ff, r0, r1);
+ } else if (q2_native_kernels(nfmt->d_type)) {
// Q2_0 down (most IQ layers): the AVX-512 kernel, ggml-cpu has only a scalar one on x86
const ActQ* a2[MAXT];
for (int t = 0; t < mjobs_[e].nt; ++t) a2[t] = &sb.a2[t];
- q2_rows_any(mjobs_[e].blob + nfmt_->down_off, nfmt_->d_row, (int) (nfmt_->n_ff / 64), a2,
+ q2_rows_any(mjobs_[e].blob + nfmt->down_off, nfmt->d_row, (int) (nfmt->n_ff / 64), a2,
mjobs_[e].nt, mjobs_[e].out, r0, r1);
} else {
const void* hq[MAXT];
for (int t = 0; t < mjobs_[e].nt; ++t) hq[t] = sb.hq[t];
- native_down_rows(*nfmt_, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);
+ native_down_rows(*nfmt, mjobs_[e].blob, hq, mjobs_[e].nt, mjobs_[e].out, r0, r1);
}
r += r1 - r0;
}
diff --git a/src/prefill/prefill.cpp b/src/prefill/prefill.cpp
index bceacbda3..4c8736f20 100644
--- a/src/prefill/prefill.cpp
+++ b/src/prefill/prefill.cpp
@@ -3568,6 +3568,7 @@ bool Prefill::run_impl(const int64_t* tokens, int64_t n, int64_t pos0, std::stri
l, p0);
cudaStreamSynchronize(m.cs);
core::progress_at("reading the prompt (batched): layer", l, p0);
+ if (on_routes) on_routes(l, ids_h, T * K); // 149-prefetch: the RAM policy's prompt seed
if (cpu_maybe) { // the rows buffer, once, at its largest (an expert takes at most MAXT tokens' rows)
const size_t cap = (size_t) std::min<int64_t>(T * K, (int64_t) strata::kernels::cpu::MAXT * m.g->n_expert) * N;
if (m.cpu_rows_n < cap) {
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index aae516316..06a2281bd 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -5454,7 +5454,10 @@ int main(int argc, char** argv) {
std::fprintf(stderr, "strata generate: file tier io prefetch ON (STRATA_IO_PREFETCH=1): whole-blob preads, "
"the layer's uncached experts and the predicted ones read by %s I/O threads (STRATA_IO_PF_THREADS, default 8)\n",
th != nullptr ? th : "8");
- else
+ if (src.io_prefetch() && src.io_sched())
+ std::fprintf(stderr, "strata generate: io prefetch scheduling ON (STRATA_IO_PF_SCHED=1): a deadline per read, "
+ "confidence admission (STRATA_IO_PF_MINCONF), a per-window byte budget (STRATA_IO_PF_BUDGET_MB / _FRAC)\n");
+ if (!src.io_prefetch())
std::fprintf(stderr, "strata generate: STRATA_IO_PREFETCH=1 is not available here (Linux mapped file tier "
"only, not with unbuffered reads); the file tier reads as before\n");
if (src.io_prefetch()) { // recommend, never force: the engine runs as asked
@@ -7868,6 +7871,23 @@ int main(int argc, char** argv) {
"cache (STRATA_PREFILL_HELPER_SOURCE)\n");
}
strata::prefill::Prefill sp;
+ // 149-prefetch STRATA_RAM_ADAPT2=1 (with STRATA_RAM_ADAPT=n, the promotions staged per window): see the window loop
+ struct RamAdapt2 {
+ bool on = false;
+ float prompt_w = 0.25f, cached_w = 0.15f;
+ int per_layer = 2;
+ int skip_top = 2;
+ int64_t installed = 0;
+ std::vector<float> seed;
+ } ram_adapt2;
+ {
+ const char* v = std::getenv("STRATA_RAM_ADAPT2");
+ ram_adapt2.on = v != nullptr && std::atoi(v) != 0;
+ if (const char* w = std::getenv("STRATA_RAM_ADAPT_PROMPT_W")) ram_adapt2.prompt_w = (float) std::max(0.0, std::atof(w));
+ if (const char* w = std::getenv("STRATA_RAM_ADAPT_CACHED_W")) ram_adapt2.cached_w = (float) std::clamp(std::atof(w), 0.0, 1.0);
+ if (const char* w = std::getenv("STRATA_RAM_ADAPT_PER_LAYER")) ram_adapt2.per_layer = std::max(1, std::atoi(w));
+ if (const char* w = std::getenv("STRATA_RAM_ADAPT_SKIP_TOP")) ram_adapt2.skip_top = std::max(0, std::atoi(w));
+ }
// the pool is idle while a prompt is read unless batch slots decode between its parts; with
// STRATA_PREFILL_CPU_SHARE the staged-chunk limit before the chunk below sizes the loans (bytes_needed reads it)
// #1607: Windows, the commit limit (RAM + page file) is what ends the engine silently; compare what this run plans
@@ -10367,6 +10387,19 @@ int main(int argc, char** argv) {
return true;
};
sp.should_stop = [&] { return stop_req.load(); };
+ // 149-prefetch STRATA_RAM_ADAPT2: the prompt's routing seeds the RAM policy with a limited weight
+ // (STRATA_RAM_ADAPT_PROMPT_W, default 0.25: an expert every prompt token routes to counts like ~5 decode routings),
+ // fading by 5% a decode window; the decode's own counts and the GPU tier are untouched
+ if (ram_adapt2.on && ram_adapt2.prompt_w > 0.0f) {
+ ram_adapt2.seed.assign((size_t) (g.n_layers * g.n_expert), 0.0f);
+ sp.on_routes = [&](int64_t l, const int32_t* ids, int64_t n) {
+ if (l < 0 || l >= g.n_layers || n <= 0) return;
+ const float per = ram_adapt2.prompt_w * 20.0f * (float) g.n_expert_used / (float) n; // = w x 20 / tokens
+ float* sd = ram_adapt2.seed.data() + l * g.n_expert;
+ for (int64_t i = 0; i < n; ++i)
+ if (ids[i] >= 0 && ids[i] < g.n_expert) sd[ids[i]] += per;
+ };
+ }
// STRATA_TRACE=1: one stderr line per step of a request (the log shows where a request stops)
const bool trace = std::getenv("STRATA_TRACE") != nullptr;
auto tr = [&](const char* what, long long a = -1, long long b = -1) {
@@ -14276,6 +14309,79 @@ int main(int argc, char** argv) {
const char* e = std::getenv("STRATA_RAM_ADAPT");
return e != nullptr ? std::max(0, std::atoi(e)) : 0;
}();
+ if (ram_adapt_n > 0 && ram_adapt2.on && !first_window && !drive.d.usage.empty() && pending.empty() &&
+ (!ajob || astate == AState::Idle) && src.complement_ready() && !multi_gpu && !peer.valid() &&
+ !remote_opt && o.batch == 0) {
+ // 149-prefetch STRATA_RAM_ADAPT2=1: install the promotions whose reads have landed (window boundary:
+ // nothing reads the RAM copy now), then stage the next ones, ranked by the stall they are expected to
+ // save per byte: (decayed decode count + the prompt's seed) x the miss's cost (a cold expert costs
+ // a drive read, one in the page cache STRATA_RAM_ADAPT_CACHED_W of that), against what the evicted
+ // one is expected to cost; up to STRATA_RAM_ADAPT_PER_LAYER pairs a layer. The reads run on the
+ // source's I/O thread, not here.
+ ram_adapt2.installed += src.ram_install(host_res.data(), g.n_expert);
+ for (float& x : ram_adapt2.seed) x *= 0.95f;
+ struct Cand { float score; int32_t e; };
+ struct RamSwap2 { float gain; int32_t layer, in, out; };
+ std::vector<RamSwap2> swaps;
+ std::vector<Cand> ins, outs;
+ for (int64_t l = 0; l < g.n_layers; ++l) {
+ const float* u = drive.d.usage.data() + l * g.n_expert;
+ const float* sd = ram_adapt2.seed.empty() ? nullptr : ram_adapt2.seed.data() + l * g.n_expert;
+ const int32_t* r = host_res.data() + l * g.n_expert;
+ ins.clear();
+ outs.clear();
+ for (int32_t e = 0; e < (int32_t) g.n_expert; ++e) {
+ if (r[e] >= 0) continue; // in VRAM
+ const float ue = u[e] + (sd != nullptr ? sd[e] : 0.0f);
+ if (src.has_resident(l, e)) outs.push_back({ue, e});
+ else if (ue >= 2.0f) ins.push_back({ue, e});
+ }
+ // the GPU tier's own next picks are left to it: its round takes the layer's most used experts
+ // outside VRAM (about 2 a layer a round), and a RAM place spent on one of them is skipped at the
+ // install (measured: 60% of the promotions skipped before this rule) - STRATA_RAM_ADAPT_SKIP_TOP
+ if (ram_adapt2.skip_top > 0 && !ins.empty()) {
+ std::vector<float> du;
+ for (const Cand& c : ins) du.push_back(u[c.e]);
+ const size_t k = std::min(du.size(), (size_t) ram_adapt2.skip_top);
+ std::nth_element(du.begin(), du.begin() + (std::ptrdiff_t) (k - 1), du.end(), std::greater<float>());
+ const float top = du[k - 1];
+ size_t dropped = 0;
+ ins.erase(std::remove_if(ins.begin(), ins.end(), [&](const Cand& c) {
+ if (dropped < k && u[c.e] >= top) { ++dropped; return true; }
+ return false;
+ }), ins.end());
+ }
+ if (ins.empty() || outs.empty()) continue;
+ const size_t P = (size_t) ram_adapt2.per_layer;
+ const size_t ni = std::min(ins.size(), 2 * P), no = std::min(outs.size(), P);
+ std::partial_sort(ins.begin(), ins.begin() + (std::ptrdiff_t) ni, ins.end(),
+ [](const Cand& a, const Cand& b) { return a.score > b.score; });
+ std::partial_sort(outs.begin(), outs.begin() + (std::ptrdiff_t) no, outs.end(),
+ [](const Cand& a, const Cand& b) { return a.score < b.score; });
+ ins.resize(ni);
+ for (Cand& c : ins) { // the miss cost of the few best: cold on the drive, or in the page cache
+ if (src.ram_queued(l, c.e)) { c.score = -1.0f; continue; }
+ c.score *= src.tier_of(l, c.e) == 1 ? ram_adapt2.cached_w : 1.0f;
+ }
+ std::sort(ins.begin(), ins.end(), [](const Cand& a, const Cand& b) { return a.score > b.score; });
+ const double bytes = (double) std::max<uint64_t>(1, src.blob_bytes(l));
+ size_t oi = 0;
+ for (size_t k = 0; k < ins.size() && oi < no; ++k) {
+ while (oi < no && src.ram_queued(l, outs[oi].e)) ++oi;
+ if (oi >= no) break;
+ const float gain = ins[k].score - outs[oi].score; // the evicted one would cost a drive read
+ if (ins[k].score <= 0.0f || gain < 1.0f) break;
+ swaps.push_back({(float) (gain / bytes * 1048576.0), (int32_t) l, ins[k].e, outs[oi].e});
+ ++oi;
+ }
+ }
+ std::sort(swaps.begin(), swaps.end(), [](const RamSwap2& a, const RamSwap2& b) { return a.gain > b.gain; });
+ int staged_n = 0;
+ for (const RamSwap2& x : swaps) {
+ if (staged_n >= ram_adapt_n) break;
+ if (src.ram_stage(x.layer, x.in, x.out)) ++staged_n;
+ }
+ } else
if (ram_adapt_n > 0 && !first_window && !drive.d.usage.empty() && pending.empty() &&
(!ajob || astate == AState::Idle) && src.complement_ready() && !multi_gpu && !peer.valid() &&
!remote_opt && o.batch == 0) {
@@ -14763,6 +14869,11 @@ int main(int argc, char** argv) {
"VRAM tier, %lld blob reads from the file\n",
(double) src.resident_bytes() / 1073741824.0, (long long) src.exchanges(),
(long long) src.file_reads());
+ if (ram_adapt2.on) // 149-prefetch: the staged promotions, cumulative
+ std::fprintf(stderr, "strata serve: resident RAM (STRATA_RAM_ADAPT2): %lld promotions read on the I/O thread, "
+ "%lld installed, %lld skipped at the install (%lld of them: the GPU tier took one of the pair)\n",
+ (long long) src.ram_staged_reads(),
+ (long long) ram_adapt2.installed, (long long) (src.ram_install_skips() + src.ram_install_gpu()), (long long) src.ram_install_gpu());
if (src.ram_adapts() > 0) // STRATA_RAM_ADAPT, cumulative
std::fprintf(stderr, "strata serve: resident RAM: %lld experts moved in from the file in place of "
"unused ones (STRATA_RAM_ADAPT)\n", (long long) src.ram_adapts());
@@ -14814,12 +14925,39 @@ int main(int argc, char** argv) {
(unsigned long long) (ic.pf_dropped - ic_prev.pf_dropped),
(double) (ic.crit_us - ic_prev.crit_us) / 1e3,
(unsigned long long) (ic.crit_n - ic_prev.crit_n));
+ if (src.io_prefetch()) // 149-prefetch: bytes read ahead, of them used before their demand; the scheduler's drops
+ std::fprintf(stderr, "; read ahead %.1f MB, %.1f MB of it used; %llu late, %llu not admitted (confidence), "
+ "%llu (budget %.0f MB/window)",
+ (double) (ic.pf_bytes - ic_prev.pf_bytes) / 1e6,
+ (double) (ic.pf_used_bytes - ic_prev.pf_used_bytes) / 1e6,
+ (unsigned long long) (ic.pf_late - ic_prev.pf_late),
+ (unsigned long long) (ic.pf_rej_conf - ic_prev.pf_rej_conf),
+ (unsigned long long) (ic.pf_rej_budget - ic_prev.pf_rej_budget),
+ (double) ic.pf_budget_bytes / 1048576.0);
std::fprintf(stderr, "\n");
io_prev = io_now;
ic_prev = ic;
fb_prev = fb;
}
}
+ if (drive.d.cold_mode != 0) { // 149-cold STRATA_COLD_OVERLAP: this request's layers that had a blob to read
+ static strata::core::ExpertDispatch cp;
+ const strata::core::ExpertDispatch& c = drive.d;
+ const double nl = (double) std::max<int64_t>(1, c.cold_layers - cp.cold_layers);
+ std::fprintf(stderr, "strata serve: cold reads (STRATA_COLD_OVERLAP=%d) this request: %lld experts from the mapping "
+ "directly; %lld layers with a blob to "
+ "read (%lld such experts, %lld ready beside them, %lld pool batches); per such layer: "
+ "R %.3f ms, C_r %.3f ms, C_c %.3f ms, the decode waited %.3f ms; total waited %.1f ms, min(R, C_r) %.1f ms\n",
+ c.cold_mode, (long long) (c.cold_zc - cp.cold_zc), (long long) (c.cold_layers - cp.cold_layers),
+ (long long) (c.cold_pending - cp.cold_pending), (long long) (c.cold_ready - cp.cold_ready),
+ (long long) (c.cold_batches - cp.cold_batches),
+ (c.ms_cold_r - cp.ms_cold_r) / nl, (c.ms_cold_cr - cp.ms_cold_cr) / nl,
+ (c.ms_cold_cc - cp.ms_cold_cc) / nl, (c.ms_cold_wait - cp.ms_cold_wait) / nl,
+ c.ms_cold_wait - cp.ms_cold_wait, c.ms_cold_hide - cp.ms_cold_hide);
+ cp.cold_layers = c.cold_layers; cp.cold_pending = c.cold_pending; cp.cold_ready = c.cold_ready; cp.cold_zc = c.cold_zc;
+ cp.cold_batches = c.cold_batches; cp.ms_cold_r = c.ms_cold_r; cp.ms_cold_cr = c.ms_cold_cr;
+ cp.ms_cold_cc = c.ms_cold_cc; cp.ms_cold_wait = c.ms_cold_wait; cp.ms_cold_hide = c.ms_cold_hide;
+ }
// STRATA_SPLIT_TIMING: where each verify stage's host time went, cumulative per window since the start
// (waiting for its GPU to ring a layer, the CPU pool and plan per layer, staging the window)
if (static const bool st_timing = std::getenv("STRATA_SPLIT_TIMING") != nullptr; st_timing)