Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 win: RAM expert store (unbuffered reads, policy, lookahead fill), Windows direct reads, dense dequant for the chunked prompt, SMT option, store test
Niko1221e7b3f1524044
diff --git a/src/dsv41/ds4_gguf.cu b/src/dsv41/ds4_gguf.cuindex 55d849448..5bee58804 100644--- a/src/dsv41/ds4_gguf.cu+++ b/src/dsv41/ds4_gguf.cu@@ -1154,6 +1154,10 @@ bool mv(const x3::MvJob* jobs, int n, cudaStream_t s) {}void rows(const x3::Mat& m, uint16_t* dst, bool bf16, cudaStream_t s) {+ if (m.fmt == kFmtGG && m.raw != nullptr) { // a K-quant / i-quant dense matrix (the IST-DAS sets' Q5_K): its own dequantizer+ if (!iq::dequant_rows(m.K, m.raw, row_stride(m), m.k, m.n, dst, bf16, s)) fail("rows: a low-bit type not covered");+ return;+ }if ((m.fmt != kFmtBF16 && m.fmt != kFmtQ8_0) || m.raw == nullptr) {fail("rows: not a BF16 or Q8_0 matrix");return;diff --git a/src/dsv41/ds4_iq.cu b/src/dsv41/ds4_iq.cuindex a49e2cffb..ed251059d 100644--- a/src/dsv41/ds4_iq.cu+++ b/src/dsv41/ds4_iq.cu@@ -3,6 +3,7 @@// src/ds4/ds4_iq.cu - a GGUF's low-bit routed experts on the device (see ds4_iq.cuh). The block layouts and grids are// ggml's (ggml-common.h, MIT, third_party/ggml/LICENSE); the sign and grid tricks are llama.cpp's vecdotq.cuh.#include "ds4_iq.cuh"+#include "ds4_numerics.cuh"#include <cuda_fp16.h>#if !defined(STRATA_USE_HIP)@@ -1007,8 +1008,60 @@ void mv_nc(const Jobs& J, int nj, int max_n, int max_c, cudaStream_t s) {else mv_kernel<T, 16><<<grid, kThreads, 0, s>>>(J);}++// ---------------------------------------------------------------- whole matrices as 16-bit rows (the chunked prompt's GEMMs)+// n rows of k values from `raw` (ld bytes a row): one thread a 32-value sub-block, the sub-block's codes and scales from decode<>+// (value = d s q - m of its half, ggml's dequantize_row_*), rounded to BF16 (bf16) or F16+template <int T>+__global__ void __launch_bounds__(kThreads) dq_rows_kernel(const uint8_t* __restrict__ raw, size_t ld, int k, int n,+ uint16_t* __restrict__ dst, bool bf16) {+ __shared__ uint32_t tab[Ty<T>::TAB > 0 ? Ty<T>::TAB : 1];+ load_tab<T>(tab);+ __syncthreads();+ const int nsb = k / 32;+ const size_t total = (size_t) n * (size_t) nsb;+ for (size_t i = (size_t) blockIdx.x * blockDim.x + threadIdx.x; i < total; i += (size_t) gridDim.x * blockDim.x) {+ const size_t r = i / (size_t) nsb;+ const int kb = (int) (i - r * (size_t) nsb);+ Sub sb;+ sb.m0 = sb.m1 = 0.0f;+ decode<T>(raw + r * ld, kb, tab, sb);+ const float d0 = __fmul_rn(sb.d, (float) sb.s0), d1 = __fmul_rn(sb.d, (float) sb.s1);+ uint16_t* o = dst + r * (size_t) k + (size_t) kb * 32;+#pragma unroll+ for (int j = 0; j < 32; ++j) {+ const int q = (int) (int8_t) ((uint32_t) sb.q[j >> 2] >> (8 * (j & 3)));+ const float v = j < 16 ? __fsub_rn(__fmul_rn(d0, (float) q), sb.m0) : __fsub_rn(__fmul_rn(d1, (float) q), sb.m1);+ o[j] = bf16 ? strata::ds4::bf16_rn(v) : __half_as_ushort(__float2half_rn(v));+ }+ }+}+} // namespace+bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s) {+ if (raw == nullptr || k % 256 != 0 || n <= 0) return false;+ const unsigned blocks = (unsigned) std::min<size_t>(((size_t) n * (size_t) (k / 32) + kThreads - 1) / kThreads, 65535);+#define DS4_DQ(T) dq_rows_kernel<T><<<blocks, kThreads, 0, s>>>((const uint8_t*) raw, ld, k, n, dst, bf16)+ switch (type) {+ case kIQ2_XXS: DS4_DQ(kIQ2_XXS); break;+ case kIQ2_XS: DS4_DQ(kIQ2_XS); break;+ case kIQ2_S: DS4_DQ(kIQ2_S); break;+ case kIQ3_XXS: DS4_DQ(kIQ3_XXS); break;+ case kIQ3_S: DS4_DQ(kIQ3_S); break;+ case kIQ4_XS: DS4_DQ(kIQ4_XS); break;+ case kIQ1_M: DS4_DQ(kIQ1_M); break;+ case kQ4_K: DS4_DQ(kQ4_K); break;+ case kQ5_K: DS4_DQ(kQ5_K); break;+ case kQ6_K: DS4_DQ(kQ6_K); break;+ case kQ2_K: DS4_DQ(kQ2_K); break;+ case kQ3_K: DS4_DQ(kQ3_K); break;+ default: return false;+ }+#undef DS4_DQ+ return cudaGetLastError() == cudaSuccess;+}+bool supported(int t) {// the types the kernels below are instantiated for (DS4_IQ_SWITCH + the trellis ones + the dense K-quants)switch (t) {diff --git a/src/dsv41/ds4_iq.cuh b/src/dsv41/ds4_iq.cuhindex 0cf49c957..62dc1b574 100644--- a/src/dsv41/ds4_iq.cuh+++ b/src/dsv41/ds4_iq.cuh@@ -116,6 +116,10 @@ struct MvJob {};bool mv(const MvJob* jobs, int n, cudaStream_t s);+/// n rows of k values of a matrix of the type (ld bytes a row) as BF16 (bf16) or F16 bits, dst[n][k]: the chunked prompt path's+/// dense GEMM operand. false: a type not covered.+bool dequant_rows(int type, const void* raw, size_t ld, int k, int n, uint16_t* dst, bool bf16, cudaStream_t s);+/// The trellis types' kernels (ds4_tq.cu), reached from the entry points above for those types: arithmetic as the CPU lane's/// (ds4_tq_cpu.cpp: int8 activations with an F32 scale a 32, exact weights), on the tensor cores of gfx12 (WMMA) or in FMAs.namespace tq {diff --git a/src/dsv41/ds4_main.cpp b/src/dsv41/ds4_main.cppindex 889f30405..be716c935 100644--- a/src/dsv41/ds4_main.cpp+++ b/src/dsv41/ds4_main.cpp@@ -71,7 +71,7 @@ std::vector<int> parse_ids(const std::string& s) {void usage() {std::fprintf(stderr,"maya-ds4 --pack <model>/maya-ds4 [--gpus 0,1,...] [--max-context N] [--reserve-mib N]\n"- " [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--no-expert-cache] [--verbose]\n"+ " [--vram-experts F] [--cpu-threads N] [--expert-profile FILE|none] [--ssd-cache GB] [--ssd-policy lru|lfu|lfulru] [--no-expert-cache] [--verbose]\n"" [--tp N] [--threads on|off] [--prefill-chunk N] [--dspark on|off|none] [--dspark-conf F]\n"" [--vision on|off] [--mmap-experts on|off|auto]\n"" (--ids I,J,... [--images FILE] [--n N] [--dump-logits FILE] | --serve\n"@@ -666,6 +666,11 @@ int main(int argc, char** argv) {else if (a == "--vram-experts") o.vram_experts = std::atof(val("--vram-experts").c_str());else if (a == "--cpu-threads") o.cpu_threads = std::atoi(val("--cpu-threads").c_str());else if (a == "--expert-profile") o.profile = val("--expert-profile");+ else if (a == "--ssd-cache") o.ssd_cache_gb = std::atof(val("--ssd-cache").c_str());+ else if (a == "--ssd-policy") {+ const std::string v = val("--ssd-policy");+ o.ssd_policy = v == "lfu" ? 1 : v == "lfulru" ? 2 : 0;+ }else if (a == "--eos") eos = val("--eos");else if (a == "--no-expert-cache") o.expert_cache = false;else if (a == "--dspark") {@@ -813,7 +818,7 @@ int main(int argc, char** argv) {out.push_back(tok);std::printf("%d,", tok);std::fflush(stdout);- if (tok == 1 || (int) out.size() >= n) break;+ if ((tok == 1 && std::getenv("DS4_NO_EOS") == nullptr) || (int) out.size() >= n) break; // (DS4_NO_EOS: decode n tokens whatever)if (got.empty()) {int off = 0, acc = 0;const bool try_spec = spec && !cool;@@ -870,6 +875,18 @@ int main(int argc, char** argv) {std::fprintf(stderr, "decode left out %.1f experts a token not in RAM (%.1f%% of the routed)\n",(double) (s.skipped - s0.skipped) / nt,100.0 * (double) (s.skipped - s0.skipped) / std::max<double>(1.0, (double) (s.experts - s0.experts)));+ if (s.store_slots_max > 0) { // --ssd-cache+ const double h = (double) (s.store_hits - s0.store_hits), m = (double) (s.store_misses - s0.store_misses);+ std::fprintf(stderr,+ "store: %.1f%% of the CPU lane's experts were in RAM (%.1f hits, %.1f misses a token); read %.1f MB a token "+ "(demand and guesses), the CPU lane waited %.1f ms a token for them; %lld prefetched, %lld used, %lld given up "+ "unused; slots %lld of %lld, %lld evictions, %lld failed reads\n",+ 100.0 * h / std::max(1.0, h + m), h / nt, m / nt, (double) (s.store_bytes - s0.store_bytes) / 1e6 / nt,+ (s.store_wait_ms - s0.store_wait_ms) / nt, (long long) (s.store_prefetched - s0.store_prefetched),+ (long long) (s.store_pf_used - s0.store_pf_used), (long long) (s.store_pf_wasted - s0.store_pf_wasted),+ (long long) s.store_slots, (long long) s.store_slots_max, (long long) (s.store_evictions - s0.store_evictions),+ (long long) (s.store_fails - s0.store_fails));+ }if (s.ahead_picks > s0.ahead_picks) { // lookaheadconst double p = (double) (s.ahead_picks - s0.ahead_picks), c = (double) (s.ahead_cold - s0.ahead_cold);std::fprintf(stderr, "lookahead: the guesses a layer before held %.1f%% of the picks (their top k %.1f%%, two layers "diff --git a/src/dsv41/ds4_model.cu b/src/dsv41/ds4_model.cuindex 774cfce73..8829c8594 100644--- a/src/dsv41/ds4_model.cu+++ b/src/dsv41/ds4_model.cu@@ -20,6 +20,7 @@#include "ds4_kernels.cuh"#include "ds4_os.hpp"#include "ds4_pool.hpp"+#include "ds4_store.hpp"#include "ds4_vision.cuh"#include "strata/dsv41/exl3_cpu.hpp"#include "strata/dsv41/exl3.hpp"@@ -111,7 +112,11 @@ std::vector<int> cpu_list(const std::string& s) { // "0-3,8,10-11"// each NUMA node's physical cores (the first logical CPU of each core), in orderstd::vector<std::vector<int>> node_cores() {#if defined(_WIN32)- std::vector<std::vector<int>> nodes = os::node_cores();+ static const bool smt = [] { // DS4_CPU_SMT=1: the second hardware thread of every core as well (memory-bound rows gain from it)+ const char* v = std::getenv("DS4_CPU_SMT");+ return v != nullptr && std::atoi(v) != 0;+ }();+ std::vector<std::vector<int>> nodes = os::node_cores(smt);#elsestd::vector<std::vector<int>> nodes;for (int n = 0; n < 64; ++n) {@@ -981,6 +986,8 @@ struct Model::Impl {const char* v = std::getenv("STRATA_DS4_DISK_SKIP_FROM");return v != nullptr ? std::max(0, std::atoi(v)) : 0;}();+ // the RAM store (--ssd-cache): the CPU lane's experts in slots a policy owns, read unbuffered (null: off)+ std::unique_ptr<ExpertStore> store;bool in_ram(const Layer& l, int e) const { // (DISK_SKIP_W) expert e's pages in RAM nowif (!l.ram.empty() && l.ram[(size_t) e] >= 0) return true;if (l.lay.fmt == gg::kFmtMXFP4) {@@ -1586,6 +1593,10 @@ struct Model::Impl {return true;}const uint8_t* mx_mat(const Layer& l, int e, int r) const { // expert e's matrix r: its RAM-tier slot, else the file's+ if (store) {+ const int s = store->find(l.il, e);+ if (s >= 0) return store->mat(s, r);+ }if (!l.ram.empty() && l.ram[(size_t) e] >= 0) // (a copy with NH > 1: part 0's rows only - whole: read_mx)return copy_mode ? ram_part[(size_t) l.ram[(size_t) e]][0] + cp_lay[(size_t) l.il][(size_t) r]: ram_slots[(size_t) l.ram[(size_t) e]] + l.lay.trellis[r];@@ -2389,6 +2400,7 @@ struct Model::Impl {const uint64_t avail = ram_avail();warm_bytes = avail > kRamMargin ? std::min<uint64_t>(avail - kRamMargin, (uint64_t) need) : 0;if (avail == ~uint64_t{0}) warm_bytes = (uint64_t) need;+ if (store_gb() > 0) warm_bytes = 0; // (the store is the RAM tier: nothing to warm in the OS cache)std::fprintf(stderr,"ds4: the experts VRAM leaves over (~%.1f GB) are read in place from the model's files%s ""(%.1f GB of RAM free to keep them: the most-routed %.1f GB read ahead; prompts in chunks of %d)\n",@@ -2416,6 +2428,7 @@ struct Model::Impl {if (!place_experts(err)) return false;if (!load_experts(err)) return false;if (pk.gguf) make_pools();+ init_store();if (chunk > 0) {// the CPU lane's parts pinned (the chunks stream them to the GPUs by DMA), each layer's virtual slotsconst double tr = now_ms();@@ -4171,6 +4184,46 @@ struct Model::Impl {return s.str();}+ double store_gb() const {+ if (!pk.gguf) return 0.0;+ if (const char* v = std::getenv("DS4_STORE_GB")) return std::atof(v);+ return o.ssd_cache_gb;+ }+ void init_store() {+ const double gb = store_gb();+ if (gb <= 0.0 || L.empty() || L[0].lay.fmt != gg::kFmtMXFP4 || !file_experts) return;+ std::vector<ExpertStore::LayerGeo> geo((size_t) g.nl); // (the trunk's layers: the drafter's bigger experts stay in place)+ for (const Layer& l : L) {+ if (l.il >= g.nl) continue;+ ExpertStore::LayerGeo& gl = geo[(size_t) l.il];+ gl.n_exp = l.n_exp;+ for (int r = 0; r < 3; ++r) {+ gl.file[r] = l.mx_file[r];+ gl.off[r] = l.mx_off[r];+ gl.stride[r] = l.mx_stride[r];+ gl.bytes[r] = mx_part_bytes(l, r);+ }+ }+ std::vector<std::string> paths;+ ExpertStore::Cfg cfg;+ cfg.budget_bytes = (size_t) (gb * 1e9);+ cfg.threads = std::getenv("DS4_STORE_THREADS") ? std::atoi(std::getenv("DS4_STORE_THREADS")) : 8;+ cfg.floor_bytes = (size_t) (std::getenv("DS4_STORE_FLOOR_GB") ? std::atof(std::getenv("DS4_STORE_FLOOR_GB")) : 8.0) << 30;+ cfg.policy = (ExpertStore::Policy) (std::getenv("DS4_STORE_POLICY") ? std::atoi(std::getenv("DS4_STORE_POLICY")) : o.ssd_policy);+ cfg.ram_avail = [] { return ram_avail(); };+ bool direct = true;+ for (os::fd_t f : pk.dfds) direct = direct && f != os::kBadFd;+ auto st = std::make_unique<ExpertStore>();+ if (!st->init(cfg, geo, pk.fds, pk.dfds, pk.pfds)) {+ std::fprintf(stderr, "ds4: --ssd-cache: the store could not start\n");+ return;+ }+ store = std::move(st);+ std::fprintf(stderr, "ds4: RAM store: up to %zu slots of %.1f MB (%.1f GB), policy %d, %d reader threads, %s reads\n",+ store->slots_max(), store->slot_bytes() / 1e6, store->slots_max() * (double) store->slot_bytes() / 1e9,+ (int) cfg.policy, cfg.threads, direct ? "unbuffered" : "BUFFERED (no direct handle)");+ }+int cpu_threads() const {int n = 0;for (const auto& pl : pools) n += pl->size() - 1;@@ -4599,7 +4652,7 @@ struct Model::Impl {// more than computing a warm expert, 0.34 ms a 6.3 MB matrix on Windows)std::vector<std::pair<const void*, size_t>> want;for (const Ent& en : ents)- if (l.ram.empty() || l.ram[(size_t) en.e] < 0)+ if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});os::prefetch_cold(want);}@@ -4689,7 +4742,7 @@ struct Model::Impl {{ // (os::prefetch, as above)std::vector<std::pair<const void*, size_t>> want;for (const Ent& en : ents)- if (l.ram.empty() || l.ram[(size_t) en.e] < 0)+ if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});os::prefetch(want);}@@ -4747,10 +4800,31 @@ struct Model::Impl {run_nodes(ne * DB, f);}+ // the window's CPU-lane experts made ready (and pinned) in the store for the length of the call+ struct StoreHold {+ Impl* m = nullptr;+ int layer = 0;+ std::vector<int> es;+ std::vector<char> ok;+ StoreHold(Impl* m_, const Layer& l, const std::vector<Ent>& ents) {+ if (!m_->store) return;+ m = m_;+ layer = l.il;+ for (const Ent& en : ents)+ if ((l.ram.empty() || l.ram[(size_t) en.e] < 0) && std::find(es.begin(), es.end(), en.e) == es.end()) es.push_back(en.e);+ m->store->ensure(layer, es.data(), (int) es.size(), ok);+ }+ ~StoreHold() {+ if (m) m->store->release(layer, es.data(), (int) es.size(), ok);+ }+ };void cpu_experts(const Layer& l, const float* x, const std::vector<Ent>& ents, size_t ldx = 0,const int* ord = nullptr) {if (ldx == 0) ldx = (size_t) g.dim;- if (l.lay.fmt == gg::kFmtMXFP4) return cpu_experts_mx(l, x, ents, ldx, ord);+ if (l.lay.fmt == gg::kFmtMXFP4) {+ StoreHold hold(this, l, ents);+ return cpu_experts_mx(l, x, ents, ldx, ord);+ }const int E = g.dim, FF = g.ff, FFh = FF / NH, Eh = E / NH, GQ = E / 256, DQ = FF / 128;const int GB = FFh / 128, DB = Eh / 128, EB = E / 128, FB = FF / 128;const int ne = (int) ents.size();@@ -5215,7 +5289,9 @@ struct Model::Impl {if (a == 1 && (rk[e] == 0 || rk[e] > i + 1)) rk[e] = (uint8_t) (i + 1);if ((fl[e] & 8) || !file_read(n, e)) continue;fl[e] |= 8;- for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});+ if (store) store->prefetch(j, e);+ else+ for (int q = 0; q < 3; ++q) pf_want.push_back({mx_mat(n, e, q), mx_part_bytes(n, q)});++st.ahead_pf;}}@@ -7930,7 +8006,25 @@ void Model::checkpoint(int slot) { p_->checkpoint(slot); }bool Model::restore(int64_t limit) { return p_->restore(limit); }int64_t Model::restorable(int64_t limit) const { return p_->restorable(limit); }void Model::save_profile() const { p_->save_profile(); }-const Stats& Model::stats() const { return p_->st; }+const Stats& Model::stats() const {+ Stats& st = p_->st;+ if (p_->store) { // (the store keeps its own counters: copied in when asked)+ const ExpertStore& x = *p_->store;+ st.store_hits = x.stats().hits.load();+ st.store_misses = x.stats().misses.load();+ st.store_prefetched = x.stats().prefetched.load();+ st.store_pf_used = x.stats().prefetch_used.load();+ st.store_pf_wasted = x.stats().prefetch_wasted.load();+ st.store_evictions = x.stats().evictions.load();+ st.store_bytes = x.stats().bytes_read.load();+ st.store_fails = x.stats().fails.load();+ st.store_wait_ms = x.stats().wait_ns.load() / 1e6;+ st.store_busy_ms = x.busy_ms();+ st.store_slots = (int64_t) x.slots_allocated();+ st.store_slots_max = (int64_t) x.slots_max();+ }+ return st;+}std::string Model::describe() const { return p_->describe(); }std::vector<std::string> Model::info_lines() const { return p_->info_lines(); }diff --git a/src/dsv41/ds4_model.hpp b/src/dsv41/ds4_model.hppindex 91c72f771..41b87a847 100644--- a/src/dsv41/ds4_model.hpp+++ b/src/dsv41/ds4_model.hpp@@ -32,6 +32,11 @@ struct Options {// the CPU lane's experts read in place from the model's files, the page cache keeping what fits, instead of// copied into RAM: 1 on, 0 off, -1 (auto) where the RAM this process can get does not hold themint mmap_experts = -1;+ // the experts the VRAM leaves over are cached in RAM by a store of their own (ds4_store.hpp): whole experts read unbuffered into+ // slots, a replacement policy (0 lru, 1 lfu, 2 lfu then lru), filled on demand and by the lookahead's guesses. GB of RAM at+ // most (never past the RAM the process can still get less 8 GB); 0: off, the OS page cache as before+ double ssd_cache_gb = 0.0;+ int ssd_policy = 0;bool dspark = true; // load the DSpark drafter (the checkpoint's mtp.* blocks) for speculative decodingfloat dspark_conf = 0.5f; // a draft is verified while its confidence (sigmoid) is at least this// tensor parallelism: runs of tp GPUs of one kind (the same memory) take the same layers, each a share of the@@ -73,6 +78,12 @@ struct Stats {int64_t ahead_picks = 0, ahead_hit_k = 0, ahead_hit = 0, ahead_hit2 = 0;int64_t ahead_cold = 0, ahead_cold_hit = 0, ahead_pf = 0;int64_t ahead_rank_n[16] = {}, ahead_rank_hit[16] = {}; // the next-layer guesses by rank, and how many were picked+ // the RAM store (--ssd-cache), totals since the load: reads asked for and found there (hits), missed (misses), started ahead+ // of use (prefetched) of which used / given up unused, slots given up, bytes read, ms the CPU lane waited for the reads+ int64_t store_hits = 0, store_misses = 0, store_prefetched = 0, store_pf_used = 0, store_pf_wasted = 0, store_evictions = 0;+ int64_t store_bytes = 0, store_fails = 0;+ double store_wait_ms = 0.0, store_busy_ms = 0.0;+ int64_t store_slots = 0, store_slots_max = 0;int64_t drafts_offered = 0, drafts_accepted = 0, drafts_rounds = 0;// per draft position (the 1st .. 8th of a round): verified with all the drafts before it accepted, and acceptedint64_t draft_reached[8] = {}, draft_kept[8] = {};diff --git a/src/dsv41/ds4_os.hpp b/src/dsv41/ds4_os.hppindex 689f5ec79..d93d9632b 100644--- a/src/dsv41/ds4_os.hpp+++ b/src/dsv41/ds4_os.hpp@@ -61,8 +61,16 @@ inline fd_t open_ro(const std::string& path) {/// kBadFd where the filesystem refuses it, or on Windows, whose unbuffered reads of a mapped file take a slow path)inline fd_t open_direct(const std::string& path) {#if defined(_WIN32)- (void) path;- return kBadFd;+ // FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED: the disk straight into an aligned buffer, any number of reads in flight on+ // one handle (DS4_WIN_DIRECT=0: none, the page cache's buffered reads as before)+ static const bool off = [] {+ const char* v = std::getenv("DS4_WIN_DIRECT");+ return v != nullptr && std::atoi(v) == 0;+ }();+ if (off) return kBadFd;+ HANDLE h = CreateFileW(wide(path).c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,+ FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, nullptr);+ return h == INVALID_HANDLE_VALUE ? kBadFd : (fd_t) h;#elseconst int fd = ::open(path.c_str(), O_RDONLY | O_DIRECT);return fd < 0 ? kBadFd : (fd_t) fd;@@ -78,8 +86,40 @@ inline uint64_t direct_span(uint64_t off, uint64_t n) {}inline bool pread_direct(fd_t f, void* dst, uint64_t n, uint64_t off) {#if defined(_WIN32)- (void) f, (void) dst, (void) n, (void) off;- return false;+ struct Ev { // (each thread its own, closed when it ends)+ HANDLE h = CreateEventW(nullptr, TRUE, FALSE, nullptr);+ ~Ev() {+ if (h != nullptr) CloseHandle(h);+ }+ };+ static thread_local Ev ev;+ if (ev.h == nullptr) return false;+ const uint64_t a0 = off & ~(kDirectAlign - 1), need = direct_span(off, n), want = (off - a0) + n;+ uint8_t* p = (uint8_t*) dst;+ uint64_t got = 0;+ while (got < need) {+ const DWORD ask = (DWORD) std::min<uint64_t>(need - got, 1ull << 30); // (a multiple of 4096)+ OVERLAPPED ov{};+ const uint64_t at = a0 + got;+ ov.Offset = (DWORD) (at & 0xffffffffu);+ ov.OffsetHigh = (DWORD) (at >> 32);+ ov.hEvent = ev.h;+ ResetEvent(ev.h);+ DWORD r = 0;+ if (!ReadFile((HANDLE) f, p + got, ask, nullptr, &ov)) {+ const DWORD e = GetLastError();+ if (e == ERROR_HANDLE_EOF) break;+ if (e != ERROR_IO_PENDING) return false;+ }+ if (!GetOverlappedResult((HANDLE) f, &ov, &r, TRUE)) {+ if (GetLastError() == ERROR_HANDLE_EOF) break;+ return false;+ }+ if (r == 0) break; // (the file's end: the last aligned read is short)+ got += r;+ if (r % kDirectAlign != 0) break;+ }+ return got >= want;#elseconst uint64_t a0 = off & ~(kDirectAlign - 1), need = direct_span(off, n), want = (off - a0) + n;uint8_t* p = (uint8_t*) dst;@@ -370,7 +410,7 @@ inline void pin_thread(std::thread& t, int cpu) {#if defined(_WIN32)/// Each NUMA node's physical cores (the first logical CPU of each), processor group 0 - Windows' answer to/// /sys/devices/system/node; empty if the OS does not say.-inline std::vector<std::vector<int>> node_cores() {+inline std::vector<std::vector<int>> node_cores(bool smt = false) { // (smt: every logical CPU of a core, not its first)std::vector<std::vector<int>> nodes;auto query = [](LOGICAL_PROCESSOR_RELATIONSHIP rel, std::vector<uint8_t>& buf) {DWORD len = 0;@@ -398,7 +438,12 @@ inline std::vector<std::vector<int>> node_cores() {break;}for (size_t n = 0; n < masks.size() && first >= 0; ++n)- if ((masks[n] >> first) & 1) nodes[n].push_back(first);+ if ((masks[n] >> first) & 1) {+ nodes[n].push_back(first);+ if (smt)+ for (int b = first + 1; b < 64; ++b)+ if ((m >> b) & 1) nodes[n].push_back(b);+ }}o += e->Size;}diff --git a/src/dsv41/ds4_store.hpp b/src/dsv41/ds4_store.hppnew file mode 100644index 000000000..1a407ba83--- /dev/null+++ b/src/dsv41/ds4_store.hpp@@ -0,0 +1,329 @@+// Strata. src/dsv41/ds4_store.hpp - the RAM tier of the DeepSeek-V4.1 routed experts as a cache of its own: whole experts read+// from the model's files by unbuffered requests (the OS page cache neither used nor disturbed), kept in a fixed arena of slots+// that a replacement policy owns, filled on demand (the CPU lane's picks) and ahead of time (the lookahead's guesses).+//+// An expert (layer, e) is three matrices at three file offsets (the GGUF's one tensor a role and layer). A slot holds their+// bytes at 4096-aligned places (unbuffered reads land the file range rounded out to 4096 at an aligned address; the matrix's+// first byte is `lead` bytes into it), so mat(slot, r) is the pointer the CPU lane's rows read - the same bytes as the file's+// mapping, so every answer is the same. Nothing here touches a device.+//+// Memory: slots are allocated in chunks as they are first needed (VirtualAlloc / mmap, private memory, not pinned), never past+// `max_slots` nor past the RAM this process can still get less `floor_bytes` (the owner's session and the system keep theirs).+//+// Policy (pick the slot to give up, among the unpinned ready ones): lru (oldest touch), lfu (lowest decayed hit count), or+// lfu with an LRU tie-break. A slot whose expert a window is computing is pinned from ensure() to release().+#pragma once++#include "ds4_os.hpp"+#include "ds4_pool.hpp"++#include <algorithm>+#include <atomic>+#include <chrono>+#include <cmath>+#include <cstdint>+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <functional>+#include <memory>+#include <mutex>+#include <string>+#include <vector>++namespace strata::ds4 {++class ExpertStore {+public:+ struct LayerGeo {+ int n_exp = 0;+ int file[3] = {-1, -1, -1};+ uint64_t off[3] = {0, 0, 0}, stride[3] = {0, 0, 0}; // matrix r of expert e: file[r], off[r] + e stride[r]+ uint64_t bytes[3] = {0, 0, 0};+ };+ enum Policy { kLru = 0, kLfu = 1, kLfuLru = 2 };+ struct Stats {+ std::atomic<int64_t> hits{0}, misses{0}, prefetched{0}, prefetch_used{0}, prefetch_wasted{0}, evictions{0}, fails{0};+ std::atomic<int64_t> bytes_read{0}, wait_ns{0};+ };++ struct Cfg {+ const std::vector<std::string>* paths = nullptr; // the files, by the layers' file indices+ int threads = 8;+ size_t budget_bytes = 0; // the arena at most (grown in chunks as slots are first needed)+ size_t floor_bytes = 8ull << 30; // RAM left alone+ Policy policy = kLru;+ float decay = 0.97f; // lfu: heat x decay every `decay_every` accesses of its layer+ int decay_every = 240;+ std::function<uint64_t()> ram_avail; // the RAM this process can still get (~0: unknown)+ };++ bool init(const Cfg& c, const std::vector<LayerGeo>& layers, const std::vector<os::fd_t>& fds, const std::vector<os::fd_t>& dfds,+ const std::vector<os::fd_t>& pfds) {+ cfg_ = c;+ geo_ = layers;+ fds_ = fds;+ dfds_ = dfds;+ pfds_ = pfds;+ slot_bytes_ = 0;+ for (const LayerGeo& g : geo_) {+ size_t need = 0;+ for (int r = 0; r < 3; ++r) need += (size_t) g.bytes[r] + 2 * os::kDirectAlign;+ slot_bytes_ = std::max(slot_bytes_, need);+ }+ slot_bytes_ = (slot_bytes_ + os::kDirectAlign - 1) & ~(size_t) (os::kDirectAlign - 1);+ if (slot_bytes_ == 0 || c.budget_bytes < slot_bytes_) return false;+ nmax_ = c.budget_bytes / slot_bytes_;+ slots_.reset(new Slot[nmax_]);+ where_.resize(geo_.size());+ heat_.resize(geo_.size());+ hits_in_layer_.assign(geo_.size(), 0);+ for (size_t l = 0; l < geo_.size(); ++l) {+ where_[l].assign((size_t) geo_[l].n_exp, -1);+ heat_[l].assign((size_t) geo_[l].n_exp, 0.0f);+ }+ io_.reset(new IoQueue(std::max(1, c.threads)));+ return true;+ }+ ~ExpertStore() {+ io_.reset();+ for (auto& ch : chunks_) os::anon_free(ch.first, ch.second);+ }++ size_t slot_bytes() const { return slot_bytes_; }+ size_t slots_allocated() const { return nalloc_; }+ size_t slots_max() const { return nmax_; }+ const Stats& stats() const { return st_; }+ double busy_ms() const { return io_ ? io_->busy_ms() : 0.0; }++ /// The slot holding (layer, e) ready to read, or -1. (Any thread; a slot found stays valid while it is pinned.)+ int find(int layer, int e) const {+ if (layer < 0 || layer >= (int) where_.size()) return -1; // (a layer the store does not cover: the DSpark drafter's)+ const int s = where_[(size_t) layer][(size_t) e];+ if (s < 0) return -1;+ Slot& sp = slots_[(size_t) s];+ try_ready(sp);+ return sp.state.load(std::memory_order_acquire) == kReady && sp.layer == layer && sp.expert == e ? s : -1;+ }+ const uint8_t* mat(int slot, int r) const { return slots_[(size_t) slot].base + slots_[(size_t) slot].lead[r]; }++ /// Makes the experts es[0..n) of `layer` ready and pinned (reading what is not here), waiting for the reads; the+ /// caller releases them. False if a read failed or no slot could be had for one (then none of the failed ones is+ /// pinned: ok[i] says which are; the caller reads those another way).+ bool ensure(int layer, const int* es, int n, std::vector<char>& ok) {+ ok.assign((size_t) n, 0);+ if (layer < 0 || layer >= (int) geo_.size()) return false;+ std::vector<int> sl((size_t) n, -1);+ {+ std::lock_guard<std::mutex> lk(mu_);+ for (int i = 0; i < n; ++i) {+ const int e = es[i];+ touch_heat(layer, e);+ int s = where_[(size_t) layer][(size_t) e];+ if (s >= 0) {+ Slot& sp = slots_[(size_t) s];+ try_ready(sp);+ if (sp.state.load() == kReady && !sp.prefetch) st_.hits.fetch_add(1, std::memory_order_relaxed);+ if (sp.prefetch) st_.prefetch_used.fetch_add(1, std::memory_order_relaxed);+ sp.prefetch = false;+ sp.pin++;+ sp.tick = ++clock_;+ sl[(size_t) i] = s;+ continue;+ }+ st_.misses.fetch_add(1, std::memory_order_relaxed);+ s = start_fill(layer, e, true);+ sl[(size_t) i] = s;+ }+ }+ const auto t0 = std::chrono::steady_clock::now();+ bool all = true;+ for (int i = 0; i < n; ++i) {+ const int s = sl[(size_t) i];+ if (s < 0) {+ all = false;+ continue;+ }+ Slot& sp = slots_[(size_t) s];+ while (sp.left.load(std::memory_order_acquire) > 0) io_->wait_any(2000);+ std::lock_guard<std::mutex> lk(mu_);+ finish(sp);+ if (sp.state.load() == kReady) ok[(size_t) i] = 1;+ else {+ sp.pin--;+ sl[(size_t) i] = -1;+ all = false;+ }+ }+ st_.wait_ns.fetch_add(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count(),+ std::memory_order_relaxed);+ return all;+ }+ void release(int layer, const int* es, int n, const std::vector<char>& ok) {+ if (layer < 0 || layer >= (int) geo_.size()) return;+ std::lock_guard<std::mutex> lk(mu_);+ for (int i = 0; i < n; ++i)+ if (ok[(size_t) i]) {+ const int s = where_[(size_t) layer][(size_t) es[i]];+ if (s >= 0 && slots_[(size_t) s].pin > 0) slots_[(size_t) s].pin--;+ }+ }+ /// Starts reading (layer, e) if it is neither here nor on its way (a guess: unpinned, may be given up before it is used).+ void prefetch(int layer, int e) {+ if (layer < 0 || layer >= (int) geo_.size() || e < 0 || e >= geo_[(size_t) layer].n_exp) return;+ if (where_[(size_t) layer][(size_t) e] >= 0) return;+ std::lock_guard<std::mutex> lk(mu_);+ if (where_[(size_t) layer][(size_t) e] >= 0) return;+ if (start_fill(layer, e, false) >= 0) st_.prefetched.fetch_add(1, std::memory_order_relaxed);+ }+ /// Heat of an expert as the policy sees it (tests / the report).+ float heat(int layer, int e) const { return heat_[(size_t) layer][(size_t) e]; }++private:+ static constexpr int kFree = 0, kLoading = 1, kReady = 2, kFailed = 3;+ struct Slot {+ uint8_t* base = nullptr;+ uint32_t lead[3] = {0, 0, 0};+ int layer = -1, expert = -1, pin = 0;+ bool prefetch = false;+ std::atomic<int> state{kFree}, left{0};+ std::atomic<bool> bad{false};+ uint64_t tick = 0;+ };++ void touch_heat(int layer, int e) {+ heat_[(size_t) layer][(size_t) e] += 1.0f;+ if (cfg_.policy != kLru && ++hits_in_layer_[(size_t) layer] % (uint64_t) cfg_.decay_every == 0) {+ // (the layer's heat x decay: a linear pass over 384 floats every 240 accesses of that layer)+ for (float& h : heat_[(size_t) layer]) h *= cfg_.decay;+ }+ }+ // a slot for (layer, e) and its reads started; -1 if there is none to be had. mu_ held.+ int start_fill(int layer, int e, bool demand) {+ int s = take_slot(demand);+ if (s < 0) {+ st_.fails.fetch_add(1, std::memory_order_relaxed);+ return -1;+ }+ Slot& sp = slots_[(size_t) s];+ if (sp.layer >= 0 && sp.expert >= 0) {+ where_[(size_t) sp.layer][(size_t) sp.expert] = -1;+ if (sp.prefetch) st_.prefetch_wasted.fetch_add(1, std::memory_order_relaxed);+ st_.evictions.fetch_add(1, std::memory_order_relaxed);+ }+ sp.layer = layer;+ sp.expert = e;+ sp.pin = demand ? 1 : 0;+ sp.prefetch = !demand;+ sp.tick = ++clock_;+ sp.state.store(kLoading, std::memory_order_release);+ sp.bad.store(false);+ sp.left.store(3, std::memory_order_release);+ where_[(size_t) layer][(size_t) e] = s;+ const LayerGeo& g = geo_[(size_t) layer];+ IoQueue::Job jobs[3];+ size_t at = 0;+ for (int r = 0; r < 3; ++r) {+ const uint64_t off = g.off[r] + (uint64_t) e * g.stride[r];+ const uint64_t lead = off % os::kDirectAlign;+ sp.lead[r] = (uint32_t) (at + lead);+ IoQueue::Job& j = jobs[r];+ j.fd = pfds_[(size_t) g.file[r]];+ j.dfd = dfds_[(size_t) g.file[r]];+ j.off = off;+ j.len = g.bytes[r];+ j.dst = sp.base + at;+ j.left = &sp.left;+ j.bad = &sp.bad;+ at += os::direct_span(off, g.bytes[r]);+ }+ st_.bytes_read.fetch_add((int64_t) (g.bytes[0] + g.bytes[1] + g.bytes[2]), std::memory_order_relaxed);+ io_->push(jobs, 3);+ return s;+ }+ // loading -> ready once its reads are done and whole (lock-free; a failed one is cleaned up by finish / take_slot)+ void try_ready(Slot& sp) const {+ int want = kLoading;+ if (sp.left.load(std::memory_order_acquire) == 0 && !sp.bad.load(std::memory_order_acquire))+ sp.state.compare_exchange_strong(want, kReady, std::memory_order_acq_rel);+ }+ void finish(Slot& sp) { // mu_ held; reads done+ if (sp.state.load() != kLoading) return;+ if (sp.bad.load()) {+ where_[(size_t) sp.layer][(size_t) sp.expert] = -1;+ sp.layer = sp.expert = -1;+ sp.state.store(kFree, std::memory_order_release);+ sp.prefetch = false;+ st_.fails.fetch_add(1, std::memory_order_relaxed);+ return;+ }+ sp.state.store(kReady, std::memory_order_release);+ }+ bool grow() {+ if (nalloc_ >= nmax_) return false;+ constexpr size_t kChunk = 64; // slots+ const size_t n = std::min(kChunk, nmax_ - nalloc_);+ const size_t bytes = n * slot_bytes_;+ const uint64_t avail = cfg_.ram_avail ? cfg_.ram_avail() : ~uint64_t{0};+ if (avail != ~uint64_t{0} && avail < bytes + cfg_.floor_bytes) return false;+ void* p = os::anon_alloc(bytes, -1, 1);+ if (p == nullptr) return false;+ chunks_.push_back({p, bytes});+ for (size_t i = 0; i < n; ++i) slots_[nalloc_ + i].base = (uint8_t*) p + i * slot_bytes_;+ nalloc_ += n;+ return true;+ }+ // the slot to give up (or a free one): the policy's lowest among the unpinned ready ones. mu_ held.+ int take_slot(bool demand) {+ for (size_t i = free_hint_; i < nalloc_; ++i)+ if (slots_[i].state.load(std::memory_order_acquire) == kFree) {+ free_hint_ = i + 1;+ return (int) i;+ }+ if (grow()) return take_slot(demand);+ int best = -1;+ double bv = 0;+ for (size_t i = 0; i < nalloc_; ++i) {+ Slot& sp = slots_[i];+ if (sp.state.load(std::memory_order_acquire) == kLoading && sp.left.load(std::memory_order_acquire) == 0) {+ try_ready(sp);+ if (sp.state.load(std::memory_order_acquire) == kLoading) finish(sp); // (failed: freed)+ if (sp.state.load(std::memory_order_acquire) == kFree) {+ free_hint_ = std::min(free_hint_, i);+ return (int) i;+ }+ }+ if (sp.pin > 0 || sp.state.load(std::memory_order_acquire) != kReady) continue;+ double v;+ if (cfg_.policy == kLru) v = (double) sp.tick;+ else {+ const double h = heat_[(size_t) sp.layer][(size_t) sp.expert];+ v = cfg_.policy == kLfu ? h : h * 1e12 + (double) sp.tick; // lfu with the age as tie-break+ if (cfg_.policy == kLfuLru) v = h + 1e-9 * (double) sp.tick;+ }+ if (best < 0 || v < bv) {+ best = (int) i;+ bv = v;+ }+ }+ if (best < 0) return -1;+ free_hint_ = std::min<size_t>(free_hint_, (size_t) best);+ return best;+ }++ Cfg cfg_;+ std::vector<LayerGeo> geo_;+ std::vector<os::fd_t> fds_, dfds_, pfds_;+ size_t slot_bytes_ = 0, nmax_ = 0, nalloc_ = 0, free_hint_ = 0;+ std::unique_ptr<Slot[]> slots_;+ std::vector<std::pair<void*, size_t>> chunks_;+ std::vector<std::vector<int>> where_;+ std::vector<std::vector<float>> heat_;+ std::vector<uint64_t> hits_in_layer_;+ uint64_t clock_ = 0;+ std::mutex mu_;+ std::unique_ptr<IoQueue> io_;+ Stats st_;+};++} // namespace strata::ds4diff --git a/tests/dsv41/store_test.cpp b/tests/dsv41/store_test.cppnew file mode 100644index 000000000..4181db041--- /dev/null+++ b/tests/dsv41/store_test.cpp@@ -0,0 +1,86 @@+// Strata. ExpertStore (src/dsv41/ds4_store.hpp) on real expert files, no device: the bytes a slot holds equal the file's, the+// unbuffered reads' rate, the policy's hits on a skewed random access stream.+// ds4_store_test <gguf> <layer0 gate off> <layer0 up off> <layer0 down off> <expert bytes (gate/up/down)> [budget GB] [policy 0|1|2] [accesses]+// (offsets from the GGUF's tensor directory: tools/dsv41 header scan; the three tensors of a layer are [384][...] so expert e of+// a matrix lies at off + e * bytes)+#include "ds4_store.hpp"++#include <chrono>+#include <cstdio>+#include <cstdlib>+#include <random>++using namespace strata::ds4;++int main(int argc, char** argv) {+ if (argc < 6) {+ std::printf("usage: ds4_store_test <gguf> <gate off> <up off> <down off> <matrix bytes> [budget GB] [policy] [accesses]\n");+ return 2;+ }+ const std::string path = argv[1];+ const uint64_t offs[3] = {std::strtoull(argv[2], nullptr, 10), std::strtoull(argv[3], nullptr, 10), std::strtoull(argv[4], nullptr, 10)};+ const uint64_t mb = std::strtoull(argv[5], nullptr, 10);+ const double gb = argc > 6 ? std::atof(argv[6]) : 1.0;+ const int policy = argc > 7 ? std::atoi(argv[7]) : 0;+ const int n_acc = argc > 8 ? std::atoi(argv[8]) : 2000;+ const os::fd_t fd = os::open_ro(path), dfd = os::open_direct(path), pfd = os::open_par(path);+ if (fd == os::kBadFd || pfd == os::kBadFd) {+ std::printf("cannot open %s\n", path.c_str());+ return 2;+ }+ std::printf("direct handle: %s\n", dfd != os::kBadFd ? "yes" : "NO (buffered)");+ ExpertStore::LayerGeo geo;+ geo.n_exp = 384;+ for (int r = 0; r < 3; ++r) {+ geo.file[r] = 0;+ geo.off[r] = offs[r];+ geo.stride[r] = mb;+ geo.bytes[r] = mb;+ }+ ExpertStore st;+ ExpertStore::Cfg cfg;+ cfg.budget_bytes = (size_t) (gb * 1e9);+ cfg.threads = std::getenv("STORE_TEST_THREADS") ? std::atoi(std::getenv("STORE_TEST_THREADS")) : 8;+ cfg.policy = (ExpertStore::Policy) policy;+ cfg.floor_bytes = 4ull << 30;+ cfg.ram_avail = [] { return os::ram_avail(); };+ if (!st.init(cfg, {geo}, {fd}, {dfd}, {pfd})) {+ std::printf("init failed\n");+ return 2;+ }+ std::printf("slot %.2f MB, up to %zu slots\n", st.slot_bytes() / 1e6, st.slots_max());+ // a skewed stream: 70% of the accesses from the 64 hottest experts, the rest uniform; 6 a "token"+ std::mt19937_64 rng(7);+ auto pick = [&] { return rng() % 100 < 70 ? (int) (rng() % 64) : (int) (rng() % 384); };+ std::vector<uint8_t> ref(mb);+ int bad = 0;+ long checked = 0;+ const auto t0 = std::chrono::steady_clock::now();+ for (int a = 0; a < n_acc; ++a) {+ int es[6];+ for (int i = 0; i < 6; ++i) {+ es[i] = pick();+ for (int j = 0; j < i; ++j)+ if (es[j] == es[i]) es[i] = (es[i] + 1 + j) % 384;+ }+ std::vector<char> ok;+ st.ensure(0, es, 6, ok);+ if (a % 17 == 0 && !std::getenv("STORE_TEST_NOCMP")) // a sample compared with the file read plainly+ for (int i = 0; i < 6; ++i)+ if (ok[(size_t) i])+ for (int r = 0; r < 3; ++r) {+ const int s = st.find(0, es[i]);+ os::pread_full(fd, ref.data(), mb, offs[r] + (uint64_t) es[i] * mb);+ ++checked;+ if (s < 0 || std::memcmp(st.mat(s, r), ref.data(), mb) != 0) ++bad;+ }+ st.release(0, es, 6, ok);+ }+ const double dt = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();+ const auto& s = st.stats();+ std::printf("%d accesses of 6 experts in %.2f s: hits %lld misses %lld evictions %lld fails %lld; read %.2f GB (%.2f GB/s); waited %.2f s; "+ "compared %ld matrices, %d differ\n",+ n_acc, dt, (long long) s.hits.load(), (long long) s.misses.load(), (long long) s.evictions.load(), (long long) s.fails.load(),+ s.bytes_read.load() / 1e9, s.bytes_read.load() / 1e9 / dt, s.wait_ns.load() / 1e9, checked, bad);+ return bad == 0 ? 0 : 1;+}diff --git a/tools/dsv41/win/cachesim.py b/tools/dsv41/win/cachesim.pyindex e89027823..b6c171afb 100644--- a/tools/dsv41/win/cachesim.py+++ b/tools/dsv41/win/cachesim.py@@ -22,6 +22,9 @@ import structimport sys+WARM = 0 # accesses before this index warm the cache but are not counted++def read_trace(path, k=6):"""-> list of steps; a step = list of (layer, [rows of expert ids], guesses or None)."""data = open(path, 'rb').read()@@ -85,15 +88,16 @@ class Sizes:def sim_lru(acc, size, cap):od = collections.OrderedDict()used = hits = hitb = tot = totb = 0- for _, l, e in acc:+ for i, (_, l, e) in enumerate(acc):k = (l, e)sz = size(l)- tot += 1- totb += sz+ cnt = i >= WARM+ tot += cnt+ totb += sz * cntif k in od:od.move_to_end(k)- hits += 1- hitb += sz+ hits += cnt+ hitb += sz * cntcontinueod[k] = szused += sz@@ -111,12 +115,13 @@ def sim_lfu(acc, size, cap, half_life_acc=20000):for t, (_, l, e) in enumerate(acc):k = (l, e)sz = size(l)- tot += 1- totb += sz+ cnt = t >= WARM+ tot += cnt+ totb += sz * cnt# CRF: decayed count, kept in log space: v = log(count) + lam * tif k in res:- hits += 1- hitb += sz+ hits += cnt+ hitb += sz * cntold = score.get(k)v = lam * tscore[k] = v if old is None else v + math.log1p(math.exp(old - v)) if old - v < 50 else old@@ -153,13 +158,14 @@ def pick_static(freq, size, cap):def sim_static(acc, size, chosen):hits = hitb = tot = totb = 0- for _, l, e in acc:+ for i, (_, l, e) in enumerate(acc):sz = size(l)- tot += 1- totb += sz+ cnt = i >= WARM+ tot += cnt+ totb += sz * cntif (l, e) in chosen:- hits += 1- hitb += sz+ hits += cnt+ hitb += sz * cntreturn hits, hitb, tot, totb@@ -167,19 +173,20 @@ def sim_static_lru(acc, size, cap, chosen, pinned_bytes):od = collections.OrderedDict()used = hits = hitb = tot = totb = 0room = cap - pinned_bytes- for _, l, e in acc:+ for i, (_, l, e) in enumerate(acc):k = (l, e)sz = size(l)- tot += 1- totb += sz+ cnt = i >= WARM+ tot += cnt+ totb += sz * cntif k in chosen:- hits += 1- hitb += sz+ hits += cnt+ hitb += sz * cntcontinueif k in od:od.move_to_end(k)- hits += 1- hitb += sz+ hits += cnt+ hitb += sz * cntcontinueod[k] = szused += sz@@ -203,11 +210,12 @@ def sim_belady(acc, size, cap):for i, (_, l, e) in enumerate(acc):k = (l, e)sz = size(l)- tot += 1- totb += sz+ cnt = i >= WARM+ tot += cnt+ totb += sz * cntif k in res:- hits += 1- hitb += sz+ hits += cnt+ hitb += sz * cntres[k] = nxt[i]heapq.heappush(heap, (-nxt[i], k))continue@@ -300,9 +308,11 @@ def main():ap.add_argument('--sizes', default='C:/ds-ista-work/sizes.json')ap.add_argument('--caps', default='20,30,40,44', help='cache sizes in GB (VRAM hot set + pinned RAM)')ap.add_argument('--train', nargs='*', default=[], help='traces the static profile is built from (others than the test)')+ ap.add_argument('--warm-steps', type=int, default=0, help='steps at the start that fill the cache but are not counted')ap.add_argument('--skip', default='', help='policies to skip (comma): lru,lfu,static,static+lru,belady,ahead')a = ap.parse_args()size = Sizes(a.sizes)+ global WARMskip = set(a.skip.split(','))train_acc = []for t in a.train:@@ -311,6 +321,12 @@ def main():steps = read_trace(path)acc = accesses(steps)nsteps = len(steps)+ WARM = 0+ if a.warm_steps:+ WARM = next((i for i, x in enumerate(acc) if x[0] >= a.warm_steps), len(acc))+ nsteps -= a.warm_steps+ uniq = len({(l, e) for _, l, e in acc})+ print(' unique experts touched: %d of 15360 (%.1f%%), %.1f GB' % (uniq, 100 * uniq / 15360, sum(size(l) for l, e in {(l, e) for _, l, e in acc}) / 1e9))print('== %s: %d steps, %d expert accesses (%.1f a step)' % (os.path.basename(path), nsteps, len(acc), len(acc) / max(1, nsteps)))tb = sum(size(l) for _, l, _ in acc) / nstepsprint(' bytes a step (union per layer): %.2f GB' % (tb / 1e9))