Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

fix: drop the pack and MTP files' page cache before the NUMA placement

Baptiste Rajaut393eec80f942
diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hpp
index 600295844..3ab77dc2a 100644
--- a/include/strata/core/expert_source.hpp
+++ b/include/strata/core/expert_source.hpp
@@ -716,6 +716,9 @@ public:
bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, std::string& err);
/// CS-T: the --native shard (native_experts.txt names the other files beside it); see `open`.
void set_gguf(const std::string& native) { gguf_ = native; }
+ /// STRATA_NUMA_SPLIT: files the engine read at start outside this source (the pack's dense.bin, the MTP head),
+ /// whose cached pages the placement's drop gives back too. Set BEFORE `pin_cache_complement`.
+ void set_extra_drop_files(std::vector<std::string> paths) { extra_drop_files_ = std::move(paths); }
/// Whether the experts are read from the GGUF shards in place (no experts.bin).
bool gguf_mode() const { return !role_ptr_.empty(); }
/// Pin a compact host mirror of experts absent from a fully filled static GPU cache. The mmap remains open
@@ -1062,6 +1065,7 @@ private:
static constexpr uint64_t kNoComplement = detail::kNoCacheComplement;
// ---- CS-T: the GGUF shards in place
std::string gguf_;
+ std::vector<std::string> extra_drop_files_; ///< set_extra_drop_files
struct Map {
const uint8_t* base = nullptr;
uint64_t bytes = 0;
diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp
index 537671a91..5ac52d151 100644
--- a/src/core/expert_source.cpp
+++ b/src/core/expert_source.cpp
@@ -3704,6 +3704,17 @@ bool FileExpertSource::pin_cache_complement(
};
if (maps_.empty()) drop(base_, mapped_bytes_, fd_);
else for (const Map& m : maps_) drop(m.closed ? nullptr : m.base, m.bytes, m.fd);
+ // the files read at start outside this source (set_extra_drop_files): their pages are already copied to VRAM
+ // or RAM, and they sat on whichever node read them (~2.2 GiB on node 0 measured on the 3090 box)
+ // CBA: whole files by path; a file read at start but not listed stays cached (the per-node free line shows
+ // it), a registry of the files the loaders open if that ever bites
+ for (const std::string& p : extra_drop_files_) {
+ const int fd = ::open(p.c_str(), O_RDONLY | O_CLOEXEC);
+ if (fd >= 0) {
+ (void) posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
+ ::close(fd);
+ }
+ }
};
#endif
if (bytes > 0) {
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index 7366427a0..b30d45c08 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -7990,6 +7990,17 @@ int main(int argc, char** argv) {
}
src.stage_lend_regions(std::move(stage_lends));
}
+ { // the pack's and the MTP head's files: read at start, their cache is given back by the complement's drop
+ std::vector<std::string> drop;
+ for (const std::string& d : {o.pack, o.mtp}) {
+ std::error_code ec;
+ if (d.empty()) continue;
+ if (std::filesystem::is_regular_file(d, ec)) { drop.push_back(d); continue; }
+ for (const auto& f : std::filesystem::directory_iterator(d, ec))
+ if (f.is_regular_file(ec)) drop.push_back(f.path().string());
+ }
+ src.set_extra_drop_files(std::move(drop));
+ }
const std::vector<std::pair<int32_t, int32_t>>& rank_all = profile_all.empty() ? profile : profile_all;
bool resident_ok = src.pin_cache_complement(xcache, err, o.resident_pin, stage_pairs, lend_from,
o.resident_headroom, o.resident_budget, &rank_all);