Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
fix: drop the pack and MTP files' page cache before the NUMA placement
Baptiste Rajaut393eec80f942
diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hppindex 600295844..3ab77dc2a 100644--- a/include/strata/core/expert_source.hpp+++ b/include/strata/core/expert_source.hpp@@ -716,6 +716,9 @@ public:bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err);/// CS-T: the --native shard (native_experts.txt names the other files beside it); see `open`.void set_gguf(const std::string& native) { gguf_ = native; }+ /// STRATA_NUMA_SPLIT: files the engine read at start outside this source (the pack's dense.bin, the MTP head),+ /// whose cached pages the placement's drop gives back too. Set BEFORE `pin_cache_complement`.+ void set_extra_drop_files(std::vector<std::string> paths) { extra_drop_files_ = std::move(paths); }/// Whether the experts are read from the GGUF shards in place (no experts.bin).bool gguf_mode() const { return !role_ptr_.empty(); }/// Pin a compact host mirror of experts absent from a fully filled static GPU cache. The mmap remains open@@ -1062,6 +1065,7 @@ private:static constexpr uint64_t kNoComplement = detail::kNoCacheComplement;// ---- CS-T: the GGUF shards in placestd::string gguf_;+ std::vector<std::string> extra_drop_files_; ///< set_extra_drop_filesstruct Map {const uint8_t* base = nullptr;uint64_t bytes = 0;diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cppindex 537671a91..5ac52d151 100644--- a/src/core/expert_source.cpp+++ b/src/core/expert_source.cpp@@ -3704,6 +3704,17 @@ bool FileExpertSource::pin_cache_complement(};if (maps_.empty()) drop(base_, mapped_bytes_, fd_);else for (const Map& m : maps_) drop(m.closed ? nullptr : m.base, m.bytes, m.fd);+ // the files read at start outside this source (set_extra_drop_files): their pages are already copied to VRAM+ // or RAM, and they sat on whichever node read them (~2.2 GiB on node 0 measured on the 3090 box)+ // CBA: whole files by path; a file read at start but not listed stays cached (the per-node free line shows+ // it), a registry of the files the loaders open if that ever bites+ for (const std::string& p : extra_drop_files_) {+ const int fd = ::open(p.c_str(), O_RDONLY | O_CLOEXEC);+ if (fd >= 0) {+ (void) posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);+ ::close(fd);+ }+ }};#endifif (bytes > 0) {diff --git a/src/program/generate.cpp b/src/program/generate.cppindex 7366427a0..b30d45c08 100644--- a/src/program/generate.cpp+++ b/src/program/generate.cpp@@ -7990,6 +7990,17 @@ int main(int argc, char** argv) {}src.stage_lend_regions(std::move(stage_lends));}+ { // the pack's and the MTP head's files: read at start, their cache is given back by the complement's drop+ std::vector<std::string> drop;+ for (const std::string& d : {o.pack, o.mtp}) {+ std::error_code ec;+ if (d.empty()) continue;+ if (std::filesystem::is_regular_file(d, ec)) { drop.push_back(d); continue; }+ for (const auto& f : std::filesystem::directory_iterator(d, ec))+ if (f.is_regular_file(ec)) drop.push_back(f.path().string());+ }+ src.set_extra_drop_files(std::move(drop));+ }const std::vector<std::pair<int32_t, int32_t>>& rank_all = profile_all.empty() ? profile : profile_all;bool resident_ok = src.pin_cache_complement(xcache, err, o.resident_pin, stage_pairs, lend_from,o.resident_headroom, o.resident_budget, &rank_all);