Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

fix: refuse a lopsided NUMA split and accept only 1/test for STRATA_NUMA_SPLIT

Baptiste Rajaut2496795bd94c
diff --git a/include/strata/core/expert_source.hpp b/include/strata/core/expert_source.hpp
index fb63a1cbc..600295844 100644
--- a/include/strata/core/expert_source.hpp
+++ b/include/strata/core/expert_source.hpp
@@ -121,7 +121,8 @@ uint64_t numa_pin_free(uint64_t global_free, const std::vector<uint64_t>& node_f
/// its slice-1 bytes the rest. Page rule (absolute arena offsets): any page holding a slice-0 byte belongs to slice 0,
/// so slice-0 ranges round OUTWARDS (start down, end up) and slice-1 ranges round INWARDS (start up, end down) - this
/// also settles pages shared across role and slot boundaries. Returns three ranges in arena offsets, possibly empty
-/// (first == second).
+/// (first == second). Adjacent roles smaller than a page may give overlapping slice-0 ranges: harmless (same node, the
+/// caller coalesces them).
std::array<std::pair<uint64_t, uint64_t>, 3> numa_slice_ranges(uint64_t slot_off, const uint64_t role_bytes[3],
int slice, uint64_t page = 4096);
/// The sentence appended to a "not enough RAM" start failure: how much of the RAM is file cache (Cached + Buffers of
diff --git a/src/core/expert_source.cpp b/src/core/expert_source.cpp
index 4c4ae82fd..26c7bfe4d 100644
--- a/src/core/expert_source.cpp
+++ b/src/core/expert_source.cpp
@@ -21,6 +21,7 @@
#include <algorithm>
#include <atomic>
+#include <cassert>
#include <cerrno>
#include <chrono>
#include <cstdio>
@@ -130,6 +131,7 @@ uint64_t numa_pin_free(uint64_t global_free, const std::vector<uint64_t>& node_f
std::array<std::pair<uint64_t, uint64_t>, 3> numa_slice_ranges(uint64_t slot_off, const uint64_t role_bytes[3],
int slice, uint64_t page) {
+ assert(slice == 0 || slice == 1);
std::array<std::pair<uint64_t, uint64_t>, 3> out{};
uint64_t start = slot_off;
for (int r = 0; r < 3; ++r) {
@@ -538,6 +540,8 @@ struct NumaComplementPlacement {
std::chrono::steady_clock::time_point t0;
static std::vector<uint64_t> node_free() {
+ // CBA: stops at the first missing node directory (as free_memory_bytes does), sparse node ids give a false
+ // "at risk 0.00" line only; list the node directory if a sparse-id host ever shows it
std::vector<uint64_t> out;
for (int n = 0; n < 64; ++n) {
std::ifstream node("/sys/devices/system/node/node" + std::to_string(n) + "/meminfo");
@@ -567,6 +571,7 @@ struct NumaComplementPlacement {
/// Decides from the live pool's plan; false with a logged reason when the split is on but placement cannot run.
void init(const std::vector<uint64_t>& offsets, int64_t n_expert, const std::vector<uint64_t>& blob_bytes,
const std::vector<uint64_t>& role_bytes) {
+ // relies on the pool being built first (generate.cpp: the ExpertPool at ~5632, the complement at ~7690)
const auto& plan = strata::kernels::cpu::active_numa_split();
if (!plan.on || plan.test) return;
free_before = node_free();
@@ -580,7 +585,7 @@ struct NumaComplementPlacement {
else if (plan.nodes[k] < 0 || plan.nodes[k] >= 64) why = "a node id is out of range";
if (why != nullptr) {
std::fprintf(stderr, "FileExpertSource: WARNING: NUMA placement FAILED: %s; the complement is placed by "
- "first touch as usual (this arm is invalid)\n", why);
+ "first touch as usual (the split runs with remote reads)\n", why);
return;
}
for (int k = 0; k < 2; ++k) {
@@ -625,6 +630,7 @@ struct NumaComplementPlacement {
for (uint64_t at = std::max(it->first, lo); at < std::min(it->second, hi); at += 4096)
((volatile uint8_t*) arena)[at] = 0;
};
+ // CBA: a failed spawn terminates; fall back to an unpinned touch if it ever happens
std::thread a(touch, 0), b(touch, 1);
a.join();
b.join();
@@ -654,7 +660,7 @@ struct NumaComplementPlacement {
/// Samples one page in 64 of each slice with move_pages (null node list: returns each page's node) and logs the
/// share on the intended node, once the arena is registered (its final state); `got < bytes` means the paced loop
- /// stopped early, `registered < bytes` that the driver did not page-lock all of it: the arm then fails.
+ /// stopped early, `registered < bytes` that the driver did not page-lock all of it: the placement then fails.
void report(const uint8_t* arena, uint64_t got, uint64_t registered, uint64_t bytes, const char* stop) const {
const double secs = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
uint64_t on_node[2] = {0, 0}, sampled[2] = {0, 0}, per_node[2][64] = {}, absent[2] = {0, 0};
@@ -682,17 +688,17 @@ struct NumaComplementPlacement {
double share[2];
for (int k = 0; k < 2; ++k) share[k] = sampled[k] > 0 ? 100.0 * (double) on_node[k] / (double) sampled[k] : 0.0;
std::fprintf(stderr, "FileExpertSource: NUMA placement: node %d %.2f GiB (%.1f %% of its slices), node %d %.2f GiB "
- "(%.1f %% of its slices); free before: %s; after the drop: %s; pages taken and page-locked in %.1f s\n",
+ "(%.1f %% of its slices); free before: %s; after the drop: %s; pages taken and page-locked in %.1f s since the placement started\n",
nodes[0], share[0] / 100.0 * (double) slice_bytes[0] / 1073741824.0, share[0], nodes[1],
share[1] / 100.0 * (double) slice_bytes[1] / 1073741824.0, share[1], free_text(free_before).c_str(),
free_text(free_after).c_str(), secs);
if (got < bytes)
std::fprintf(stderr, "FileExpertSource: WARNING: NUMA placement FAILED: the paced loop stopped at %.2f of %.2f "
- "GiB (%s); the rest is first-touched by unpinned copy threads (this arm is invalid)\n",
+ "GiB (%s); the rest is first-touched by unpinned copy threads (memory not placed on its node: the split runs with remote reads)\n",
(double) got / 1073741824.0, (double) bytes / 1073741824.0, stop != nullptr ? stop : "?");
else if (registered < bytes)
std::fprintf(stderr, "FileExpertSource: WARNING: NUMA placement FAILED: only %.2f of %.2f GiB page-locked (%s); "
- "the unregistered pages can be migrated by NUMA balancing (this arm is invalid)\n",
+ "the unregistered pages can be migrated by NUMA balancing (memory not placed on its node: the split runs with remote reads)\n",
(double) registered / 1073741824.0, (double) bytes / 1073741824.0, stop != nullptr ? stop : "?");
if (pin_failed)
std::fprintf(stderr, "FileExpertSource: WARNING: NUMA placement: a toucher could not be pinned to its core\n");
@@ -3740,15 +3746,6 @@ bool FileExpertSource::pin_cache_complement(
std::fprintf(stderr, "FileExpertSource: WARNING: NUMA placement FAILED: the complement is not page-locked, "
"nothing placed\n");
}
- // STRATA_COMPLEMENT_NOHUGEPAGE=1 (a test knob: the TLB cost of 4 KiB pages without the split) acts on the paced
- // arena only
- static const bool nohuge = [] {
- const char* v = std::getenv("STRATA_COMPLEMENT_NOHUGEPAGE");
- return v != nullptr && std::strcmp(v, "1") == 0;
- }();
- if (nohuge && !paced)
- std::fprintf(stderr, "FileExpertSource: STRATA_COMPLEMENT_NOHUGEPAGE=1 ignored: the complement was not "
- "allocated in steps\n");
#endif
if (pin && !paced) {
const cudaError_t allocated = cudaHostAlloc(&arena, (size_t) bytes,
@@ -3789,14 +3786,11 @@ bool FileExpertSource::pin_cache_complement(
#endif
#if defined(__linux__)
// STRATA_NUMA_SPLIT: a 2 MiB huge page would put several slices on its toucher's node, so the arena
- // takes 4 KiB pages (one call, no VMA split); STRATA_COMPLEMENT_NOHUGEPAGE=1 does this alone
- if (arena != nullptr && (numa.place || nohuge)) {
+ // takes 4 KiB pages (one call, no VMA split)
+ if (arena != nullptr && numa.place) {
if (madvise(arena, (size_t) ((bytes + page - 1) / page * page), MADV_NOHUGEPAGE) != 0)
std::fprintf(stderr, "FileExpertSource: WARNING: madvise(MADV_NOHUGEPAGE) on the complement "
"failed (errno %d)\n", errno);
- else if (!numa.place)
- std::fprintf(stderr, "FileExpertSource: STRATA_COMPLEMENT_NOHUGEPAGE=1: the complement takes "
- "4 KiB pages\n");
}
#endif
} else {
diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cpp
index 58bdd7849..fc77f9ce4 100644
--- a/src/kernels/cpu/pool.cpp
+++ b/src/kernels/cpu/pool.cpp
@@ -613,6 +613,8 @@ void ExpertPool::diag(std::FILE* f) const {
const uint64_t h = head_.load();
uint32_t batch = (uint32_t) (h >> 32), claimed = (uint32_t) h & 0xffffu, total = (uint32_t) (h >> 16) & 0xffffu,
done = done_.load();
+ // CBA: this summed-slices branch is never run by a test (it needs a diag call mid-phase), it is a trivial sum, a mid-phase
+ // diag test if it ever misreports
if (numa_n_ > 0 && mode_ >= 5) { // a split native batch never writes head_/done_: the slices' sums instead
claimed = total = done = 0;
for (int k = 0; k < numa_n_; ++k) {
@@ -676,7 +678,13 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
// STRATA_NUMA_SPLIT: decided here, before any worker starts (a worker counts into its slice's `parked` from its
// first instruction), and never revised. Unset: the plan stays empty and nothing below runs.
NumaSplitPlan plan;
- if (const char* env = std::getenv("STRATA_NUMA_SPLIT"); env != nullptr && *env != '\0' && std::strcmp(env, "0") != 0) {
+ const char* env = std::getenv("STRATA_NUMA_SPLIT");
+ const bool env_off = env == nullptr || *env == '\0' || std::strcmp(env, "0") == 0;
+ const bool env_known = !env_off && (std::strcmp(env, "1") == 0 || std::strcmp(env, "test") == 0);
+ if (!env_off && !env_known) {
+ plan.why = std::string("unknown STRATA_NUMA_SPLIT value '") + env + "' (use 1 or test)";
+ std::fprintf(stderr, "strata cpu pool: NUMA split off: %s\n", plan.why.c_str());
+ } else if (env_known) {
plan.test = std::strcmp(env, "test") == 0;
const int threads = n_ + (host_works_ ? 1 : 0); // thread j < n_: worker j; j == n_: the host
[[maybe_unused]] auto core_of = [&](int j) {
@@ -710,15 +718,21 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
std::sort(nodes.begin(), nodes.end());
nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end());
if (nodes.size() != 2) {
- std::snprintf(why, sizeof why, "the pool's threads span %zu NUMA node(s), the prototype needs exactly 2",
+ std::snprintf(why, sizeof why, "the pool's threads span %zu NUMA node(s), the NUMA split supports exactly 2",
nodes.size());
} else {
plan.nodes[0] = nodes[0];
plan.nodes[1] = nodes[1];
+ int per[2] = {0, 0};
for (int j = 0; j < threads; ++j) {
slice[(size_t) j] = node[(size_t) j] == nodes[1] ? 1 : 0;
plan.cores[slice[(size_t) j]].push_back(core_of(j));
+ ++per[slice[(size_t) j]];
}
+ // rows stay 50/50, so a slice with under half the other's threads would be the straggler of every batch
+ if (std::min(per[0], per[1]) * 2 < std::max(per[0], per[1]))
+ std::snprintf(why, sizeof why, "slices too unbalanced (node %d: %d thread%s, node %d: %d thread%s)",
+ nodes[0], per[0], per[0] == 1 ? "" : "s", nodes[1], per[1], per[1] == 1 ? "" : "s");
}
}
#else
diff --git a/src/kernels/cpu/pool_tasks_test.cpp b/src/kernels/cpu/pool_tasks_test.cpp
index 17b5341a3..d04d8fca2 100644
--- a/src/kernels/cpu/pool_tasks_test.cpp
+++ b/src/kernels/cpu/pool_tasks_test.cpp
@@ -227,6 +227,30 @@ int main(int argc, char** argv) {
unset_env("STRATA_NUMA_SPLIT");
if (!refused) { std::fprintf(stderr, "numa-split did not refuse STRATA_EXCHANGE_ROTATE=1\n"); return 1; }
}
+ { // the real split on any host: on means two distinct nodes and cores on both slices, off always says why
+ put_env("STRATA_NUMA_SPLIT", "1");
+ bool ok = false;
+ {
+ cpu::ExpertPool pool(4, true, true);
+ const auto& plan = cpu::active_numa_split();
+ ok = plan.on ? plan.nodes[0] != plan.nodes[1] && !plan.cores[0].empty() && !plan.cores[1].empty()
+ : !plan.why.empty();
+ std::printf("pool tasks: real split %s%s\n", plan.on ? "on" : "off: ", plan.why.c_str());
+ }
+ unset_env("STRATA_NUMA_SPLIT");
+ if (!ok) { std::fprintf(stderr, "numa-split real-mode plan invariant broken\n"); return 1; }
+ // a value other than 1 / test is off, with the value in the reason; "0" and "" are off silently
+ for (const char* v : {"off", "yes"}) {
+ put_env("STRATA_NUMA_SPLIT", v);
+ bool bad = false;
+ {
+ cpu::ExpertPool pool(2, true, true);
+ bad = cpu::active_numa_split().on || cpu::active_numa_split().why.find(std::string("'") + v + "'") == std::string::npos;
+ }
+ unset_env("STRATA_NUMA_SPLIT");
+ if (bad) { std::fprintf(stderr, "numa-split did not refuse STRATA_NUMA_SPLIT=%s\n", v); return 1; }
+ }
+ }
#endif
std::printf("pool tasks: %d bitwise comparisons passed, %d of them in NUMA-split mode (%d with modes 0-4 "
"interleaved)\n", checks + split_checks + mixed_checks, split_checks + mixed_checks, mixed_checks);
diff --git a/src/program/generate.cpp b/src/program/generate.cpp
index ea7cb608d..427957b9d 100644
--- a/src/program/generate.cpp
+++ b/src/program/generate.cpp
@@ -5824,6 +5824,9 @@ int main(int argc, char** argv) {
if (pool.plan_limit()[0] != '\0') // the default count: what it was bounded by (a configured count is never changed)
std::fprintf(stderr, "strata generate: CPU pool: %d workers by default (%d physical cores, %d logical processors; bounded by %s)\n",
pool.workers(), pool.plan_cores(), pool.plan_logical(), pool.plan_limit());
+ if (srcp == &arena_src && strata::kernels::cpu::active_numa_split().on && !strata::kernels::cpu::active_numa_split().test)
+ std::fprintf(stderr, "strata generate: NUMA split on, but this model's experts come from the arena path: memory is "
+ "not placed per node (known limitation)\n");
if (pool.is_hybrid() && pool.affinity() != strata::kernels::cpu::PoolAffinity::All) {
const char* aff_str = pool.affinity() == strata::kernels::cpu::PoolAffinity::PCores ? "p-cores" :
pool.affinity() == strata::kernels::cpu::PoolAffinity::All ? "all" : "auto";