Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

NUMA pool: tried by itself on a machine with 2 or more NUMA nodes, STRATA_NUMA_POOL=0 turns it off, one node unchanged

Niko1221e4d22e7ef112
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0a28c077a..1f2a6a59a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2112,6 +2112,11 @@ if(STRATA_BUILD_TESTS)
add_executable(pool_plan_test src/kernels/cpu/pool_plan_test.cpp)
target_include_directories(pool_plan_test PRIVATE include)
add_test(NAME pool_plan_test COMMAND pool_plan_test)
+ # 0.2.2: when the pool splits per NUMA node by itself (a fake sysfs node folder, mock cores; STRATA_NUMA_POOL=0, one node = unchanged)
+ add_executable(numa_pool_test src/kernels/cpu/numa_pool_test.cpp)
+ target_include_directories(numa_pool_test PRIVATE include)
+ target_compile_features(numa_pool_test PRIVATE cxx_std_17)
+ add_test(NAME numa_pool_test COMMAND numa_pool_test)
# #3204: when the HIP start-up warm-up of hipBLAS may run (pure policy, no GPU)
add_executable(warm_policy_test src/prefill/warm_policy_test.cpp)
target_include_directories(warm_policy_test PRIVATE include)
diff --git a/include/strata/kernels/cpu/numa_pool.hpp b/include/strata/kernels/cpu/numa_pool.hpp
new file mode 100644
index 000000000..5db33846a
--- /dev/null
+++ b/include/strata/kernels/cpu/numa_pool.hpp
@@ -0,0 +1,152 @@
+// include/strata/kernels/cpu/numa_pool.hpp - when the CPU expert pool splits per NUMA node (0.2.2).
+//
+// Pure decisions, no threads and no globals, so a test can feed them a made-up machine:
+// * numa_cpu_node_count_at(dir) how many NUMA nodes with CPUs a sysfs node folder lists (Linux: /sys/devices/system/node;
+// Windows reads GetNumaHighestNodeNumber in pool.cpp; every other system: 1)
+// * pick_numa_pool(...) whether the split is tried without STRATA_NUMA_SPLIT, and the line that says so
+// * decide_numa_split(...) the split's plan for a pool whose threads sit on given cores
+//
+// One node is exactly the code path before the split existed: pick_numa_pool answers `Unchanged` with no log line and the pool
+// never reads anything else from here. STRATA_NUMA_POOL=0 is the off switch of the default; STRATA_NUMA_SPLIT=1 / test keep
+// their meaning from the contributor's branch (#3201).
+#pragma once
+
+#include "strata/kernels/cpu/pool_plan.hpp"
+
+#include <algorithm>
+#include <cctype>
+#include <cstdlib>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <functional>
+#include <string>
+#include <system_error>
+#include <vector>
+
+namespace strata::kernels::cpu {
+
+/// Whether text is "0", "off", "false" or "no" (any case): the spelling of every off switch of the pool.
+inline bool numa_switch_off(const char* v) {
+ if (v == nullptr) return false;
+ std::string s(v);
+ std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return (char) std::tolower(c); });
+ return s == "0" || s == "off" || s == "false" || s == "no";
+}
+
+/// The NUMA nodes with CPUs under a sysfs node folder (`node0`, `node1`, ... each holding a `cpulist`). A node without CPUs
+/// (CXL or other memory-only nodes) is not counted: the pool splits threads, and a node with none has none to split.
+/// A folder that is missing or has no node entries counts 0 (the caller treats 0 and 1 alike: no split).
+inline int numa_cpu_node_count_at(const std::string& node_dir) {
+ std::error_code ec;
+ int nodes = 0;
+ for (std::filesystem::directory_iterator it(node_dir, ec), end; !ec && it != end; it.increment(ec)) {
+ const std::string name = it->path().filename().string();
+ if (name.size() < 5 || name.compare(0, 4, "node") != 0) continue;
+ if (!std::all_of(name.begin() + 4, name.end(), [](unsigned char c) { return std::isdigit(c) != 0; })) continue;
+ std::ifstream f(it->path() / "cpulist");
+ std::string list;
+ std::getline(f, list);
+ if (list.find_first_of("0123456789") != std::string::npos) ++nodes;
+ }
+ return nodes;
+}
+
+enum class NumaPoolChoice {
+ Unchanged, ///< nothing new: STRATA_NUMA_SPLIT (set or not) decides as it did before 0.2.2
+ Off, ///< STRATA_NUMA_POOL=0 on a machine that matters: the split is off, whatever STRATA_NUMA_SPLIT says
+ Auto, ///< two or more NUMA nodes and nothing set: try the split by itself; if it cannot run the pool stays as it was
+};
+struct NumaPoolPick {
+ NumaPoolChoice choice = NumaPoolChoice::Unchanged;
+ std::string log; ///< the line to print ("" = print nothing)
+};
+
+/// `pool_env` is STRATA_NUMA_POOL, `split_env` STRATA_NUMA_SPLIT (nullptr or "" = unset), `nodes` the NUMA nodes with CPUs.
+/// one node, nothing set -> Unchanged, silent (today's code path to the instruction)
+/// STRATA_NUMA_POOL=0, two or more nodes -> Off, with a line that says so
+/// STRATA_NUMA_POOL=0, STRATA_NUMA_SPLIT set -> Off (the off switch wins), with a line
+/// STRATA_NUMA_SPLIT set (1, test, 0, ...) -> Unchanged: the contributor's variable keeps its old meaning
+/// two or more nodes, nothing set -> Auto
+inline NumaPoolPick pick_numa_pool(const char* pool_env, const char* split_env, int nodes) {
+ const bool split_set = split_env != nullptr && *split_env != '\0';
+ NumaPoolPick pick;
+ if (numa_switch_off(pool_env) && (nodes >= 2 || split_set)) {
+ pick.choice = NumaPoolChoice::Off;
+ pick.log = "strata cpu pool: NUMA pool off (STRATA_NUMA_POOL=0): " +
+ (nodes >= 2 ? std::to_string(nodes) + " NUMA nodes found" : std::string("one NUMA node")) +
+ ", the pool runs as before";
+ return pick;
+ }
+ if (split_set || nodes < 2) return pick;
+ pick.choice = NumaPoolChoice::Auto;
+ return pick;
+}
+
+struct NumaSplitDecision {
+ bool on = false;
+ std::string why; ///< the reason when off
+ int nodes[2] = {-1, -1}; ///< the two node ids, ascending
+ std::vector<int> slice; ///< each thread's slice (0 or 1)
+ int per[2] = {0, 0}; ///< threads per slice
+};
+
+/// The real-node split of a pool whose threads sit on `thread_core` (workers first, then the host when it drains; -1 = no core).
+/// `n_workers` tells a worker from the host in the messages; `pinned` is whether the pool pins its threads; `rotate` is
+/// STRATA_EXCHANGE_ROTATE=1 (experts move between slots, the split needs fixed ones); `node_of(core)` is the core's NUMA node or -1.
+inline NumaSplitDecision decide_numa_split(const std::vector<int>& thread_core, int n_workers, bool pinned, bool rotate,
+ const std::function<int(int)>& node_of) {
+ NumaSplitDecision d;
+ const int threads = (int) thread_core.size();
+ std::vector<int> node((size_t) threads, -1);
+ if (threads < 2) {
+ d.why = "the pool has " + std::to_string(threads) + " thread, the split needs 2 or more";
+ return d;
+ }
+ if (rotate) {
+ d.why = "STRATA_EXCHANGE_ROTATE=1 moves experts between slots, the split needs fixed slots";
+ return d;
+ }
+ if (!pinned) {
+ d.why = "the pool's threads are not pinned";
+ return d;
+ }
+ for (int j = 0; j < threads; ++j) {
+ const int c = thread_core[(size_t) j];
+ if (c < 0) {
+ d.why = j < n_workers ? "worker " + std::to_string(j) + " has no core to be pinned to" : "the host thread has no core";
+ return d;
+ }
+ if ((node[(size_t) j] = node_of(c)) < 0) {
+ d.why = "CPU " + std::to_string(c) + " has no known NUMA node";
+ return d;
+ }
+ }
+ std::vector<int> nodes = node;
+ std::sort(nodes.begin(), nodes.end());
+ nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end());
+ if (nodes.size() != 2) {
+ d.why = "the pool's threads span " + std::to_string(nodes.size()) + " NUMA node(s), the NUMA split supports exactly 2";
+ return d;
+ }
+ d.nodes[0] = nodes[0];
+ d.nodes[1] = nodes[1];
+ d.slice.assign((size_t) threads, 0);
+ for (int j = 0; j < threads; ++j) {
+ d.slice[(size_t) j] = node[(size_t) j] == nodes[1] ? 1 : 0;
+ ++d.per[d.slice[(size_t) j]];
+ }
+ if (!numa_slices_balanced(d.per[0], d.per[1])) {
+ d.why = "slices too unbalanced (node " + std::to_string(nodes[0]) + ": " + std::to_string(d.per[0]) + " thread" +
+ (d.per[0] == 1 ? "" : "s") + ", node " + std::to_string(nodes[1]) + ": " + std::to_string(d.per[1]) + " thread" +
+ (d.per[1] == 1 ? "" : "s") + ")";
+ d.nodes[0] = d.nodes[1] = -1;
+ d.slice.clear();
+ d.per[0] = d.per[1] = 0;
+ return d;
+ }
+ d.on = true;
+ return d;
+}
+
+} // namespace strata::kernels::cpu
diff --git a/src/kernels/cpu/numa_pool_test.cpp b/src/kernels/cpu/numa_pool_test.cpp
new file mode 100644
index 000000000..58092f42b
--- /dev/null
+++ b/src/kernels/cpu/numa_pool_test.cpp
@@ -0,0 +1,146 @@
+// numa_pool_test: when the CPU expert pool splits per NUMA node (include/strata/kernels/cpu/numa_pool.hpp), on mock machines.
+// No threads, no OS topology: a fake sysfs node folder in a temp directory and a made-up core -> node map. It runs anywhere.
+#include "strata/kernels/cpu/numa_pool.hpp"
+
+#include <cstdio>
+#include <ctime>
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <vector>
+
+namespace cpu = strata::kernels::cpu;
+namespace fs = std::filesystem;
+
+static int g_fail = 0;
+
+static void expect(bool ok, const std::string& what) {
+ if (!ok) {
+ ++g_fail;
+ std::fprintf(stderr, "FAIL: %s\n", what.c_str());
+ }
+}
+
+// /sys/devices/system/node with `node<i>/cpulist` files; an empty string is a memory-only node
+static fs::path make_nodes(const fs::path& root, const std::vector<std::string>& cpulists, bool noise = true) {
+ fs::remove_all(root);
+ fs::create_directories(root);
+ for (size_t i = 0; i < cpulists.size(); ++i) {
+ fs::create_directories(root / ("node" + std::to_string(i)));
+ std::ofstream(root / ("node" + std::to_string(i)) / "cpulist") << cpulists[i] << "\n";
+ }
+ if (noise) { // files and folders in that directory that are not nodes
+ std::ofstream(root / "online") << "0-1\n";
+ std::ofstream(root / "possible") << "0-1\n";
+ fs::create_directories(root / "nodes");
+ fs::create_directories(root / "node");
+ fs::create_directories(root / "nodeX");
+ }
+ return root;
+}
+
+int main() {
+ const fs::path tmp = fs::temp_directory_path() / ("strata_numa_pool_test_" + std::to_string((long long) std::time(nullptr)));
+
+ // ---- the node count of a sysfs node folder
+ expect(cpu::numa_cpu_node_count_at((tmp / "missing").string()) == 0, "no folder: 0 nodes");
+ expect(cpu::numa_cpu_node_count_at(make_nodes(tmp, {"0-23"}).string()) == 1, "one node");
+ expect(cpu::numa_cpu_node_count_at(make_nodes(tmp, {"0-11,24-35", "12-23,36-47"}).string()) == 2, "two sockets: 2 nodes");
+ expect(cpu::numa_cpu_node_count_at(make_nodes(tmp, {"0-7", "8-15", "16-23", "24-31"}).string()) == 4, "four nodes");
+ expect(cpu::numa_cpu_node_count_at(make_nodes(tmp, {"0-23", ""}).string()) == 1, "a memory-only node (CXL) is not a node of CPUs");
+ expect(cpu::numa_cpu_node_count_at(make_nodes(tmp, {"0-11", "12-23", ""}).string()) == 2, "two nodes of CPUs and a memory-only one");
+
+ // ---- pick_numa_pool: one node is today's path
+ using C = cpu::NumaPoolChoice;
+ auto pick = [](const char* pool, const char* split, int nodes) { return cpu::pick_numa_pool(pool, split, nodes); };
+ for (int nodes : {0, 1}) {
+ for (const char* pool : {(const char*) nullptr, "", "1", "garbage"}) {
+ for (const char* split : {(const char*) nullptr, ""}) {
+ const auto p = pick(pool, split, nodes);
+ expect(p.choice == C::Unchanged && p.log.empty(), "one node, nothing set: Unchanged and silent");
+ }
+ }
+ const auto zero = pick("0", nullptr, nodes);
+ expect(zero.choice == C::Unchanged && zero.log.empty(), "one node, STRATA_NUMA_POOL=0: Unchanged and silent");
+ expect(pick(nullptr, "1", nodes).choice == C::Unchanged, "one node, STRATA_NUMA_SPLIT=1: the old variable decides");
+ expect(pick(nullptr, "test", nodes).choice == C::Unchanged, "one node, STRATA_NUMA_SPLIT=test: the old variable decides");
+ }
+ {
+ const auto p = pick(nullptr, nullptr, 2);
+ expect(p.choice == C::Auto && p.log.empty(), "two nodes, nothing set: Auto");
+ expect(pick("1", nullptr, 2).choice == C::Auto, "two nodes, STRATA_NUMA_POOL=1: Auto");
+ expect(pick("", "", 2).choice == C::Auto, "two nodes, empty variables: Auto");
+ expect(pick(nullptr, nullptr, 4).choice == C::Auto, "four nodes: Auto (the decision refuses it, with a line)");
+ for (const char* off : {"0", "off", "OFF", "false", "No"}) {
+ const auto o = pick(off, nullptr, 2);
+ expect(o.choice == C::Off, std::string("two nodes, STRATA_NUMA_POOL=") + off + ": Off");
+ expect(o.log.find("STRATA_NUMA_POOL=0") != std::string::npos && o.log.find("2 NUMA nodes") != std::string::npos,
+ "the off line names the switch and the node count");
+ }
+ expect(pick("0", "1", 2).choice == C::Off, "STRATA_NUMA_POOL=0 beats STRATA_NUMA_SPLIT=1");
+ expect(pick("0", "test", 1).choice == C::Off, "STRATA_NUMA_POOL=0 beats STRATA_NUMA_SPLIT=test, even on one node");
+ expect(pick(nullptr, "1", 2).choice == C::Unchanged, "two nodes, STRATA_NUMA_SPLIT=1: the old variable decides");
+ expect(pick(nullptr, "0", 2).choice == C::Unchanged, "two nodes, STRATA_NUMA_SPLIT=0: the old variable decides (off)");
+ }
+
+ // ---- decide_numa_split on a made-up 2 x 12-core box (host core 0; cores 0-11 node 0, 12-23 node 1)
+ auto node_of = [](int core) { return core < 0 ? -1 : core < 12 ? 0 : 1; };
+ auto cores = [](std::initializer_list<int> v) { return std::vector<int>(v); };
+ { // 22 workers interleaved over the nodes + the host
+ std::vector<int> c;
+ for (int i = 0; i < 11; ++i) { c.push_back(1 + i); c.push_back(12 + i); }
+ c.push_back(0); // the host
+ const auto d = cpu::decide_numa_split(c, 22, true, false, node_of);
+ expect(d.on, "22 workers + host on 2 nodes: on");
+ expect(d.nodes[0] == 0 && d.nodes[1] == 1, "the nodes are 0 and 1");
+ expect(d.per[0] == 12 && d.per[1] == 11, "12 threads on node 0 (the host counted), 11 on node 1");
+ expect(d.slice.size() == c.size() && d.slice.back() == 0, "the host drains in slice 0");
+ expect(d.slice[0] == 0 && d.slice[1] == 1, "workers alternate between the slices");
+ }
+ { // all workers on one node: refused
+ const auto d = cpu::decide_numa_split(cores({1, 2, 3, 4, 0}), 4, true, false, node_of);
+ expect(!d.on && d.why.find("span 1 NUMA node(s)") != std::string::npos, "one node's threads only: refused with the count");
+ }
+ { // a node with under half of the other's threads: unbalanced
+ std::vector<int> c = {1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 0};
+ const auto d = cpu::decide_numa_split(c, 10, true, false, node_of);
+ expect(!d.on && d.why.find("unbalanced") != std::string::npos, "10 + 1 threads: refused as unbalanced");
+ expect(d.nodes[0] == -1 && d.slice.empty() && d.per[0] == 0, "a refused split leaves no plan behind");
+ }
+ { // three nodes
+ auto three = [](int core) { return core < 8 ? 0 : core < 16 ? 1 : 2; };
+ const auto d = cpu::decide_numa_split(cores({1, 9, 17, 2, 10, 18, 0}), 6, true, false, three);
+ expect(!d.on && d.why.find("span 3 NUMA node(s)") != std::string::npos && d.why.find("exactly 2") != std::string::npos,
+ "three nodes: refused, exactly 2 are supported");
+ }
+ {
+ const auto base = cores({1, 12, 2, 13, 0});
+ expect(cpu::decide_numa_split(base, 4, true, false, node_of).on, "4 workers + host, balanced: on");
+ const auto rot = cpu::decide_numa_split(base, 4, true, true, node_of);
+ expect(!rot.on && rot.why.find("STRATA_EXCHANGE_ROTATE=1") != std::string::npos, "rotating slots: refused");
+ const auto nopin = cpu::decide_numa_split(base, 4, false, false, node_of);
+ expect(!nopin.on && nopin.why.find("not pinned") != std::string::npos, "unpinned threads: refused");
+ const auto nocore = cpu::decide_numa_split(cores({1, -1, 2, 13}), 4, true, false, node_of);
+ expect(!nocore.on && nocore.why.find("worker 1 has no core") != std::string::npos, "a worker without a core: refused");
+ const auto nohost = cpu::decide_numa_split(cores({1, 12, -1}), 2, true, false, node_of);
+ expect(!nohost.on && nohost.why.find("host thread has no core") != std::string::npos, "a host without a core: refused");
+ auto unknown = [](int core) { return core == 12 ? -1 : 0; };
+ const auto un = cpu::decide_numa_split(base, 4, true, false, unknown);
+ expect(!un.on && un.why.find("CPU 12 has no known NUMA node") != std::string::npos, "a core with no node: refused");
+ const auto one = cpu::decide_numa_split(cores({1}), 1, true, false, node_of);
+ expect(!one.on && one.why.find("needs 2 or more") != std::string::npos, "one thread: refused");
+ }
+ { // a single-node machine fed to the decision (STRATA_NUMA_SPLIT=1 on one node): refused, never half-on
+ auto flat = [](int core) { return core < 0 ? -1 : 0; };
+ const auto d = cpu::decide_numa_split(cores({1, 2, 3, 4, 0}), 4, true, false, flat);
+ expect(!d.on, "every core on node 0: off");
+ }
+
+ fs::remove_all(tmp);
+ if (g_fail) {
+ std::fprintf(stderr, "numa_pool_test: %d failure(s)\n", g_fail);
+ return 1;
+ }
+ std::printf("numa_pool_test: OK\n");
+ return 0;
+}
diff --git a/src/kernels/cpu/pool.cpp b/src/kernels/cpu/pool.cpp
index d2a91fb9d..e3f113d33 100644
--- a/src/kernels/cpu/pool.cpp
+++ b/src/kernels/cpu/pool.cpp
@@ -3,6 +3,7 @@
#include "strata/core/progress.hpp"
#include "strata/platform/aux_cpus.hpp"
#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/cpu/numa_pool.hpp"
#include <algorithm>
#include <atomic>
@@ -544,6 +545,34 @@ int cpu_numa_node(int cpu) {
}
#endif
+// The NUMA nodes with CPUs this machine has (0.2.2: the split is tried by itself from 2 on). Linux: the sysfs node folder
+// (numa_pool.hpp); Windows: GetNumaHighestNodeNumber; any other system, or a sandbox that hides the folder: 1.
+int numa_node_count() {
+#if defined(__linux__)
+ return (std::max)(1, numa_cpu_node_count_at("/sys/devices/system/node"));
+#elif defined(_WIN32)
+ ULONG highest = 0;
+ return GetNumaHighestNodeNumber(&highest) ? (int) highest + 1 : 1;
+#else
+ return 1;
+#endif
+}
+
+// The real-node split for threads on `thread_core` (numa_pool.hpp). Linux only: the placement of the experts' memory
+// (first touch by threads pinned to a node, move_pages to check) has no Windows or macOS version.
+NumaSplitDecision decide_real_numa_split([[maybe_unused]] const std::vector<int>& thread_core, [[maybe_unused]] int n_workers,
+ [[maybe_unused]] bool pinned) {
+#if defined(__linux__)
+ // the expert source's own reading of it (expert_source.cpp): exactly "1"
+ const char* rotate = std::getenv("STRATA_EXCHANGE_ROTATE");
+ return decide_numa_split(thread_core, n_workers, pinned, rotate != nullptr && std::strcmp(rotate, "1") == 0, cpu_numa_node);
+#else
+ NumaSplitDecision d;
+ d.why = "the split needs Linux";
+ return d;
+#endif
+}
+
} // namespace
ThreadAffinity pin_current_thread(int core) {
@@ -650,37 +679,77 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
if (tasks < 0 || tasks > kMaxTasks) throw std::invalid_argument("expert pool tasks must be in 0..4096");
if (const char* e = std::getenv("STRATA_POOL_SPIN_US")) // a test knob; see kSpinBeforeSleep
spin_before_sleep_ = std::chrono::microseconds((std::max)(0, std::atoi(e)));
- // STRATA_NUMA_SPLIT, read here for the worker count (the split itself is decided below, once the count is known)
+ // STRATA_NUMA_SPLIT, read here for the worker count (the split itself is decided below, once the count is known).
+ // 0.2.2: with 2 or more NUMA nodes and neither variable set the split is tried by itself, and STRATA_NUMA_POOL=0 turns that
+ // off (numa_pool.hpp). One node: pick_numa_pool says Unchanged and nothing below differs from the pool before the split.
const char* env = std::getenv("STRATA_NUMA_SPLIT");
- const bool env_off = env == nullptr || *env == '\0' || std::strcmp(env, "0") == 0;
- const bool env_known = !env_off && (std::strcmp(env, "1") == 0 || std::strcmp(env, "test") == 0);
+ const int numa_nodes = numa_node_count();
+ const NumaPoolPick pool_pick = pick_numa_pool(std::getenv("STRATA_NUMA_POOL"), env, numa_nodes);
+ if (!pool_pick.log.empty()) std::fprintf(stderr, "%s\n", pool_pick.log.c_str());
+ if (pool_pick.choice == NumaPoolChoice::Off) env = nullptr;
+ bool auto_pool = pool_pick.choice == NumaPoolChoice::Auto; // the split without STRATA_NUMA_SPLIT: only if it can run
+ bool env_off = !auto_pool && (env == nullptr || *env == '\0' || std::strcmp(env, "0") == 0);
+ bool env_known = auto_pool || (!env_off && (std::strcmp(env, "1") == 0 || std::strcmp(env, "test") == 0));
const PoolPlanInput plan_in = pool_plan_input(topo_);
const PoolPlan wplan = plan_pool_workers(plan_in);
- // the split needs workers on both nodes: its cores are taken round-robin over the packages, so any count under
- // the core count (the default or a configured one) spans them evenly (one package: the order is kept)
- if (env_known && topo_.worker_package.size() == topo_.worker_cores.size()) {
+ // the split needs workers on both nodes: its cores are taken round-robin over the nodes (the packages when a core's node
+ // is unknown), so any count under the core count (the default or a configured one) spans them evenly (one node: the order
+ // is kept)
+ auto interleave_cores = [&] {
+ if (topo_.worker_package.size() != topo_.worker_cores.size()) return;
+ std::vector<int> key = topo_.worker_package;
+#if defined(__linux__)
+ std::vector<int> by_node;
+ for (int c : topo_.worker_cores) by_node.push_back(cpu_numa_node(c));
+ if (std::none_of(by_node.begin(), by_node.end(), [](int n) { return n < 0; })) key = by_node;
+#endif
std::vector<int> cores, pkgs;
- for (size_t i : interleave_packages(topo_.worker_package)) {
+ for (size_t i : interleave_packages(key)) {
cores.push_back(topo_.worker_cores[i]);
pkgs.push_back(topo_.worker_package[i]);
}
topo_.worker_cores = std::move(cores);
topo_.worker_package = std::move(pkgs);
- }
- if (n_workers > 0) {
- n_ = n_workers;
- } else if (topo_.is_hybrid && affinity_ != PoolAffinity::All) {
- n_ = (std::max)(1, topo_.p_cores - 1);
+ };
+ // the worker count: configured, else the hybrid P cores, else the plan (`span`: every socket counts, the split's default)
+ auto pick_workers = [&](bool span, const char** limit) -> int {
+ if (n_workers > 0) return n_workers;
+ if (topo_.is_hybrid && affinity_ != PoolAffinity::All) return (std::max)(1, topo_.p_cores - 1);
+ if (span) {
+ PoolPlanInput span_in = plan_in;
+ span_in.span_packages = true;
+ const PoolPlan sp = plan_pool_workers(span_in);
+ *limit = sp.limit;
+ return (std::min)(sp.workers, (std::max)(1, (int) topo_.worker_cores.size()));
+ }
+ *limit = wplan.limit;
+ return (std::min)(wplan.workers, (std::max)(1, (int) topo_.worker_cores.size()));
+ };
+ if (auto_pool) {
+ // try the split on a copy: when it cannot run the cores, the count and the plan stay exactly what they were
+ const CpuTopology plain_topo = topo_;
+ interleave_cores();
+ const char* try_limit = nullptr;
+ const int try_n = pick_workers(true, &try_limit);
+ std::vector<int> thread_core;
+ for (int j = 0; j < try_n; ++j)
+ thread_core.push_back(pin && j < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) j] : -1);
+ if (host_works_) thread_core.push_back(topo_.host_core);
+ const NumaSplitDecision tried = decide_real_numa_split(thread_core, try_n, pin);
+ if (!tried.on) {
+ topo_ = plain_topo;
+ auto_pool = env_known = false;
+ env_off = true;
+ std::fprintf(stderr, "strata cpu pool: NUMA pool off: %s (%d NUMA nodes found), the pool runs as before\n",
+ tried.why.c_str(), numa_nodes);
+ }
} else if (env_known) {
- // ... and the default spans every socket (the plan's other bounds stay)
- PoolPlanInput span_in = plan_in;
- span_in.span_packages = true;
- const PoolPlan sp = plan_pool_workers(span_in);
- n_ = (std::min)(sp.workers, (std::max)(1, (int) topo_.worker_cores.size()));
- plan_limit_ = sp.limit;
- } else {
- n_ = (std::min)(wplan.workers, (std::max)(1, (int) topo_.worker_cores.size()));
- plan_limit_ = wplan.limit;
+ interleave_cores();
+ }
+ {
+ const char* limit = nullptr;
+ n_ = pick_workers(env_known, &limit);
+ if (limit != nullptr) plan_limit_ = limit;
}
plan_cores_ = wplan.cores;
plan_logical_ = plan_in.logical;
@@ -700,58 +769,27 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
plan.why = std::string("unknown STRATA_NUMA_SPLIT value '") + env + "' (use 1 or test)";
std::fprintf(stderr, "strata cpu pool: NUMA split off: %s\n", plan.why.c_str());
} else if (env_known) {
- plan.test = std::strcmp(env, "test") == 0;
+ plan.test = !auto_pool && std::strcmp(env, "test") == 0;
const int threads = n_ + (host_works_ ? 1 : 0); // thread j < n_: worker j; j == n_: the host
- [[maybe_unused]] auto core_of = [&](int j) {
- if (j == n_) return topo_.host_core;
- return pin && j < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) j] : -1;
- };
std::vector<int> slice((size_t) threads, 0);
- char why[160] = "";
+ std::string why;
if (threads < 2) {
- std::snprintf(why, sizeof why, "the pool has %d thread, the split needs 2 or more", threads);
+ why = "the pool has " + std::to_string(threads) + " thread, the split needs 2 or more";
} else if (plan.test) {
for (int j = 0; j < threads; ++j) slice[(size_t) j] = j % 2;
} else {
-#if defined(__linux__)
- std::vector<int> node((size_t) threads, -1);
- // the expert source's own reading of it (expert_source.cpp): exactly "1"
- const char* rotate = std::getenv("STRATA_EXCHANGE_ROTATE");
- if (rotate != nullptr && std::strcmp(rotate, "1") == 0)
- std::snprintf(why, sizeof why, "STRATA_EXCHANGE_ROTATE=1 moves experts between slots, the split needs "
- "fixed slots");
- else if (!pin) std::snprintf(why, sizeof why, "the pool's threads are not pinned");
- for (int j = 0; j < threads && why[0] == '\0'; ++j) {
- const int c = core_of(j);
- if (c < 0 && j < n_) std::snprintf(why, sizeof why, "worker %d has no core to be pinned to", j);
- else if (c < 0) std::snprintf(why, sizeof why, "the host thread has no core");
- else if ((node[(size_t) j] = cpu_numa_node(c)) < 0)
- std::snprintf(why, sizeof why, "CPU %d has no known NUMA node", c);
+ std::vector<int> thread_core;
+ for (int j = 0; j < threads; ++j)
+ thread_core.push_back(j == n_ ? topo_.host_core
+ : pin && j < (int) topo_.worker_cores.size() ? topo_.worker_cores[(size_t) j] : -1);
+ const NumaSplitDecision d = decide_real_numa_split(thread_core, n_, pin);
+ why = d.why;
+ if (d.on) {
+ slice = d.slice;
+ plan.nodes[0] = d.nodes[0];
+ plan.nodes[1] = d.nodes[1];
+ for (int j = 0; j < threads; ++j) plan.cores[slice[(size_t) j]].push_back(thread_core[(size_t) j]);
}
- if (why[0] == '\0') {
- std::vector<int> nodes = node;
- std::sort(nodes.begin(), nodes.end());
- nodes.erase(std::unique(nodes.begin(), nodes.end()), nodes.end());
- if (nodes.size() != 2) {
- std::snprintf(why, sizeof why, "the pool's threads span %zu NUMA node(s), the NUMA split supports exactly 2",
- nodes.size());
- } else {
- plan.nodes[0] = nodes[0];
- plan.nodes[1] = nodes[1];
- int per[2] = {0, 0};
- for (int j = 0; j < threads; ++j) {
- slice[(size_t) j] = node[(size_t) j] == nodes[1] ? 1 : 0;
- plan.cores[slice[(size_t) j]].push_back(core_of(j));
- ++per[slice[(size_t) j]];
- }
- if (!numa_slices_balanced(per[0], per[1]))
- std::snprintf(why, sizeof why, "slices too unbalanced (node %d: %d thread%s, node %d: %d thread%s)",
- nodes[0], per[0], per[0] == 1 ? "" : "s", nodes[1], per[1], per[1] == 1 ? "" : "s");
- }
- }
-#else
- std::snprintf(why, sizeof why, "the split needs Linux");
-#endif
}
plan.why = why;
plan.on = plan.why.empty();
@@ -767,6 +805,9 @@ ExpertPool::ExpertPool(int n_workers, bool pin, bool host_works, PoolAffinity af
hslice_ = host_works_ ? slice[(size_t) n_] : 0;
slice_threads_.assign(2, 0);
for (int k : slice) ++slice_threads_[(size_t) k];
+ if (auto_pool)
+ std::fprintf(stderr, "strata cpu pool: NUMA pool on: %d NUMA nodes found, one slice of the pool per node "
+ "(STRATA_NUMA_POOL=0 turns it off)\n", numa_nodes);
if (plan.test)
std::fprintf(stderr, "strata cpu pool: NUMA split on (test, no real nodes): slice 0 = %d threads, "
"slice 1 = %d threads\n", slice_threads_[0], slice_threads_[1]);